00000080:00200008:3.0:1787826208.627436:0:62909:0:(file.c:5224:ll_file_seek()) START file f200b.sanity-flr:[0x200000404:0x1f2:0x0], offset: 9163072, type: SEEK_DATA 00000080:00200000:3.0:1787826208.627446:0:62909:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f2:0x0] ignore/verify layout 0/0, layout version 29 restore needed 0 00020000:00200000:3.0:1787826208.627459:0:62909:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1f2:0x0]: flr state: 3, move mirror from 1 to 1, have retried: 0, mirror count: 3 00020000:00200000:3.0:1787826208.627463:0:62909:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:3.0:1787826208.627470:0:62909:0:(lov_io.c:1253:lov_io_iter_init()) component[8] flags 0x10 00020000:00200000:3.0:1787826208.627488:0:62909:0:(lov_io.c:1253:lov_io_iter_init()) component[9] flags 0x10 00000080:00200000:3.0:1787826208.627513:0:62909:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f2:0x0] updating i_size 9163072 i_blocks 17848 00000080:00200000:3.0:1787826208.627525:0:62909:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f2:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787826208.627532:0:62909:0:(file.c:1848:ll_io_set_mirror()) f200b.sanity-flr: desiginated mirror: 2 00000080:00200000:3.0:1787826208.627535:0:62909:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f2:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:3.0:1787826208.627541:0:62909:0:(lov_io.c:1253:lov_io_iter_init()) component[8] flags 0x10 00020000:00200000:3.0:1787826208.627546:0:62909:0:(lov_io.c:1253:lov_io_iter_init()) component[9] flags 0x10 00000080:00200000:3.0:1787826208.627551:0:62909:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [2237, 18446744073709551615] 00000008:00010000:3.0:1787826208.627565:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:3.0:1787826208.627569:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da0301c30, osc lock ffff907d963b8888, flags 0x0 00000008:00010000:3.0:1787826208.627573:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:3.0:1787826208.627574:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da0301c70, osc lock ffff907d963b8618, flags 0x0 00000008:00010000:3.0:1787826208.627577:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:3.0:1787826208.627579:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da0301cb0, osc lock ffff907d963b9ad0, flags 0x0 00000008:00010000:3.0:1787826208.627581:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:3.0:1787826208.627583:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da0301cf0, osc lock ffff907d963b89c0, flags 0x0 00010000:00010000:3.0:1787826208.627593:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000027595c97/0xe5f736940357b1dc lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x318:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e981 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:3.0:1787826208.627604:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (4968448 16777215) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000027595c97/0xe5f736940357b1dc lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x318:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e981 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:3.0:1787826208.627640:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000027595c97/0xe5f736940357b1dc lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x318:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e981 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:3.0:1787826208.627649:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000027595c97/0xe5f736940357b1dc lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x318:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e981 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:3.0:1787826208.627656:0:62909:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000027595c97/0xe5f736940357b1dc lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x318:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e981 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:3.0:1787826208.627667:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000048d6e517/0xe5f736940357b1e3 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x2fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e99d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:3.0:1787826208.627675:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (4194304 16777215) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000048d6e517/0xe5f736940357b1e3 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x2fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e99d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:3.0:1787826208.627683:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000048d6e517/0xe5f736940357b1e3 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x2fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e99d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:3.0:1787826208.627690:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000048d6e517/0xe5f736940357b1e3 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x2fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e99d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:3.0:1787826208.627698:0:62909:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000048d6e517/0xe5f736940357b1e3 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x2fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e99d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:3.0:1787826208.627709:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a04a15b8/0xe5f736940357b1ea lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x301:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e98f expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:3.0:1787826208.627717:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (16777216 18446744073709551615) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a04a15b8/0xe5f736940357b1ea lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x301:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e98f expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:3.0:1787826208.627732:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a04a15b8/0xe5f736940357b1ea lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x301:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e98f expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:3.0:1787826208.630007:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a04a15b8/0xe5f736940357b1ea lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x301:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e98f expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:3.0:1787826208.630024:0:62909:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a04a15b8/0xe5f736940357b1ea lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x301:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e98f expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00010000:00010000:3.0:1787826208.630068:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000099b3a7b7/0xe5f736940357b1f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x31b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e9a4 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00000080:00200080:3.0:1787826208.662450:0:62909:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88) cmd=400866b1 arg=1 00000080:00200008:3.0:1787826208.662484:0:62909:0:(file.c:6313:ll_getattr_dentry()) START file f200b.sanity-flr:[0x200000404:0x1f2:0x0](ffff907d93315b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787826208.662509:0:62909:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88),name=f200b.sanity-flr 00000002:00010000:3.0:1787826208.662544:0:62909:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f2:0x0]) in obj [0x200000404:0x1f2:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826208.662571:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d857aa800/0xe5f736940357b1b2 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0e965 expref: -99 pid: 62909 timeout: 0 lvb_type: 3 lru_score: 13 lru_type: 1 00010000:00010000:3.0:1787826208.662590:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d857aa800/0xe5f736940357b1b2 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0e965 expref: -99 pid: 62909 timeout: 0 lvb_type: 3 lru_score: 13 lru_type: 1 00000080:00200000:3.0:1787826208.662600:0:62909:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1f2:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826208.662645:0:62909:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f2:0x0](ffff907d93315b88) 00000080:00010000:3.0:1787826208.662651:0:62909:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f2:0x0](ffff907d93315b88) for lock 0xe5f736940357b1b2 00000080:00010000:3.0:1787826208.662662:0:62909:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b1b2 from it ffff9dd1430d7cd8 00010000:00010000:3.0:1787826208.662667:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d857aa800/0xe5f736940357b1b2 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0e965 expref: -99 pid: 62909 timeout: 0 lvb_type: 3 lru_score: 13 lru_type: 1 00010000:00010000:3.0:1787826208.662678:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d857aa800/0xe5f736940357b1b2 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0e965 expref: -99 pid: 62909 timeout: 0 lvb_type: 3 lru_score: 14 lru_type: 1 00000080:00200000:3.0:1787826208.662732:0:62909:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f2:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:3.0:1787826208.662751:0:62909:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1f2:0x0]: flr state: 3, move mirror from 1 to 1, have retried: 0, mirror count: 3 00020000:00200000:3.0:1787826208.662755:0:62909:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:3.0:1787826208.662760:0:62909:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1f2:0x0] 00000008:00010000:3.0:1787826208.662792:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826208.662797:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d83fae030, osc lock ffff907d963b84e0, flags 0x400000400001002 00000008:00010000:3.0:1787826208.662805:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826208.662807:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d83fae070, osc lock ffff907d963b8138, flags 0x400000400001002 00000008:00010000:3.0:1787826208.662814:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826208.662816:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d83fae0b0, osc lock ffff907d963b8000, flags 0x400000400001002 00000008:00010000:3.0:1787826208.662822:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826208.662824:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d83fae0f0, osc lock ffff907d963b8270, flags 0x400000400001002 00000008:00010000:3.0:1787826208.662872:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826208.662874:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d83fae130, osc lock ffff907d963b95f0, flags 0x400000400001002 00000008:00010000:3.0:1787826208.662907:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826208.662909:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d83fae170, osc lock ffff907d963b9e78, flags 0x400000400001002 00000008:00010000:3.0:1787826208.662939:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826208.662941:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d83fae1b0, osc lock ffff907d963b8fd8, flags 0x400000400001002 00000008:00010000:3.0:1787826208.662966:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826208.662968:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d83fae1f0, osc lock ffff907d963b9728, flags 0x400000400001002 00000008:00010000:3.0:1787826208.662985:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826208.662986:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d83fae230, osc lock ffff907d963b9d40, flags 0x400000400001002 00010000:00010000:3.0:1787826208.663010:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e20400a0/0xe5f736940357b1b9 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x312:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e973 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:3.0:1787826208.663023:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e20400a0/0xe5f736940357b1b9 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x312:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e973 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00000020:00200000:3.0:1787826208.663038:0:62909:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826208.663042:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e20400a0/0xe5f736940357b1b9 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x312:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e973 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:3.0:1787826208.663051:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e20400a0/0xe5f736940357b1b9 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x312:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e973 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.663068:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000862c5d47/0xe5f736940357b1c0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x316:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e96c expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:3.0:1787826208.663076:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 3145727) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000862c5d47/0xe5f736940357b1c0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x316:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e96c expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00000020:00200000:3.0:1787826208.663085:0:62909:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826208.663087:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000862c5d47/0xe5f736940357b1c0 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x316:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e96c expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:3.0:1787826208.663095:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000862c5d47/0xe5f736940357b1c0 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x316:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e96c expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.663112:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000fee745a6/0xe5f736940357b1c7 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x2fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e988 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:3.0:1787826208.663127:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 3145727) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000fee745a6/0xe5f736940357b1c7 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x2fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e988 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00000020:00200000:3.0:1787826208.663136:0:62909:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826208.663138:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000fee745a6/0xe5f736940357b1c7 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x2fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e988 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:3.0:1787826208.663145:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000fee745a6/0xe5f736940357b1c7 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x2fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e988 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.663160:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008e7a0e5a/0xe5f736940357b1ce lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x317:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e97a expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:3.0:1787826208.663168:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (3145728 4194303) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008e7a0e5a/0xe5f736940357b1ce lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x317:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e97a expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00000020:00200000:3.0:1787826208.663176:0:62909:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826208.663179:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008e7a0e5a/0xe5f736940357b1ce lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x317:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e97a expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:3.0:1787826208.663187:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008e7a0e5a/0xe5f736940357b1ce lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x317:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e97a expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.663200:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000007aaddef/0xe5f736940357b1d5 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x2fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e996 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:3.0:1787826208.663210:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (3145728 4194303) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000007aaddef/0xe5f736940357b1d5 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x2fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e996 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00000020:00200000:3.0:1787826208.663217:0:62909:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826208.663221:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000007aaddef/0xe5f736940357b1d5 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x2fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e996 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 17 lru_type: 1 00010000:00010000:3.0:1787826208.663227:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000007aaddef/0xe5f736940357b1d5 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x2fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e996 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.663238:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000027595c97/0xe5f736940357b1dc lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x318:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e981 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.663247:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (4194304 16777215) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000027595c97/0xe5f736940357b1dc lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x318:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e981 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00000020:00200000:3.0:1787826208.663255:0:62909:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826208.663257:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000027595c97/0xe5f736940357b1dc lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x318:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e981 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.663265:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000027595c97/0xe5f736940357b1dc lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x318:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e981 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.663277:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000048d6e517/0xe5f736940357b1e3 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x2fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e99d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.663285:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (4194304 16777215) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000048d6e517/0xe5f736940357b1e3 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x2fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e99d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00000020:00200000:3.0:1787826208.663292:0:62909:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826208.663294:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000048d6e517/0xe5f736940357b1e3 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x2fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e99d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.663300:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000048d6e517/0xe5f736940357b1e3 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x2fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e99d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.663311:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a04a15b8/0xe5f736940357b1ea lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x301:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e98f expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.663318:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (16777216 18446744073709551615) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a04a15b8/0xe5f736940357b1ea lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x301:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e98f expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00000020:00200000:3.0:1787826208.663325:0:62909:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826208.663327:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a04a15b8/0xe5f736940357b1ea lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x301:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e98f expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.663334:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a04a15b8/0xe5f736940357b1ea lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x301:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e98f expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.663347:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000099b3a7b7/0xe5f736940357b1f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x31b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e9a4 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.663356:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (16777216 18446744073709551615) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000099b3a7b7/0xe5f736940357b1f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x31b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e9a4 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00000020:00200000:3.0:1787826208.663363:0:62909:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826208.663367:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000099b3a7b7/0xe5f736940357b1f1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x31b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e9a4 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.663374:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000099b3a7b7/0xe5f736940357b1f1 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x31b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e9a4 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00000020:00200000:3.0:1787826208.663382:0:62909:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1787826208.663396:0:62909:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f2:0x0] updating i_size 9163072 i_blocks 17848 00000020:00010000:3.0:1787826208.663401:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da61ebb40 (ffff907d8662b878/0) at cl_lock_release():245 00000020:00010000:3.0:1787826208.663405:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da61ebb40 (ffff907d8662b878/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.663408:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d83fae030 (ffff907d8662bcf8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.663413:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d83fae070 (ffff907d8662bf38/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.663416:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d83fae0b0 (ffff907d8662b368/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.663418:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d83fae0f0 (ffff907da5608638/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.663421:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d83fae130 (ffff907d83f04998/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.663424:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d83fae170 (ffff907d83f043f8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.663427:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d83fae1b0 (ffff907d83f042d8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.663430:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d83fae1f0 (ffff907d83f04ea8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.663433:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d83fae230 (ffff907d83f04908/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.663436:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da61ebb40 (ffff907d8662b878/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.663440:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d83fae030 (ffff907d8662b878/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.663444:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d83fae070 (ffff907d8662b878/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.663446:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d83fae0b0 (ffff907d8662b878/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.663449:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d83fae0f0 (ffff907d8662b878/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.663451:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d83fae130 (ffff907d8662b878/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.663453:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d83fae170 (ffff907d8662b878/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.663456:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d83fae1b0 (ffff907d8662b878/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.663458:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d83fae1f0 (ffff907d8662b878/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.663461:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d83fae230 (ffff907d8662b878/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826208.663480:0:62909:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f2:0x0] ignore/verify layout 0/1, layout version 29 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:3.0:1787826208.663522:0:62909:0:(llite_lib.c:2427:ll_setattr_raw()) START file f200b.sanity-flr:[0x200000404:0x1f2:0x0](ffff907d93315b88) current size 9163072, valid attrs 2068, mode 0, uid -1976721614, gid -1, new size 9163072, atime 1787826208.000000000, mtime 1787826208.000000000, ctime 1787826208.000000000 00010000:00010000:3.0:1787826208.663551:0:62909:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d857aa800/0xe5f736940357b1b2 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0e965 expref: -99 pid: 62909 timeout: 0 lvb_type: 3 lru_score: 14 lru_type: 1 00010000:00010000:3.0:1787826208.663573:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d857ab600/0xe5f736940357b1ab lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f2:0x0].0x0 bits 0x8/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0e957 expref: -99 pid: 62909 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1787826208.663657:0:62909:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d857aa800/0xe5f736940357b1b2 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0e965 expref: -99 pid: 62909 timeout: 0 lvb_type: 3 lru_score: 14 lru_type: 1 00010000:00010000:3.0:1787826208.663666:0:62909:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787826208.663669:0:62909:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d857aa800/0xe5f736940357b1b2 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0e965 expref: -99 pid: 62909 timeout: 0 lvb_type: 3 lru_score: 14 lru_type: 1 00000020:00100000:3.0:1787826208.663695:0:62909:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1787826208.663712:0:62909:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83525680 x1874670011328384/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826208.663750:0:62909:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:62909:1874670011328384:192.168.203.104@tcp:36:lfs.0 00000100:00100000:3.0:1787826208.663866:0:62909:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d99937f00 going to sleep for 11 seconds 00000080:00200000:3.0:1787826208.752098:0:62909:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f2:0x0] updating i_size 9163072 i_blocks 17848 00000020:00010000:3.0:1787826208.752108:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da70e7a88 (ffff907da5608488/0) at cl_lock_release():245 00000020:00010000:3.0:1787826208.752113:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da70e7a88 (ffff907da5608488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.752120:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d9102ec30 (ffff907da5608908/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787826208.752135:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000045edf04d/0xe5f736940357b206 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x2f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ea4c expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.752156:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000045edf04d/0xe5f736940357b206 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x2f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ea4c expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:3.0:1787826208.752178:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d9102ec70 (ffff907da5608128/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787826208.752184:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e2ac33c6/0xe5f736940357b20d lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x313:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ea53 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.752193:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e2ac33c6/0xe5f736940357b20d lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x313:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ea53 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:3.0:1787826208.752204:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d9102ecb0 (ffff907d8662b1b8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787826208.752210:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000093ded7dc/0xe5f736940357b214 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x314:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ea5a expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.752219:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000093ded7dc/0xe5f736940357b214 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x314:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ea5a expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:3.0:1787826208.752234:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d9102ecf0 (ffff907da56081b8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787826208.752239:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000008c16a203/0xe5f736940357b21b lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x2f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ea61 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.752246:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000008c16a203/0xe5f736940357b21b lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x2f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ea61 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:3.0:1787826208.752255:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d9102ed30 (ffff907da5608f38/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787826208.752261:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000002ff0324c/0xe5f736940357b222 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x2fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ea68 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.752269:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000002ff0324c/0xe5f736940357b222 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x2fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ea68 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:3.0:1787826208.752277:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d9102ed70 (ffff907da5608368/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787826208.752282:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000fec3b6ad/0xe5f736940357b229 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x315:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ea6f expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.752290:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000fec3b6ad/0xe5f736940357b229 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x315:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ea6f expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:3.0:1787826208.752298:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d9102edb0 (ffff907da56085a8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787826208.752303:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a49bef08/0xe5f736940357b230 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x300:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ea76 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.752317:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a49bef08/0xe5f736940357b230 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x300:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ea76 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:3.0:1787826208.752326:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d9102edf0 (ffff907da5608a28/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787826208.752332:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000888724c1/0xe5f736940357b237 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ea7d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.752340:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000888724c1/0xe5f736940357b237 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x31a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ea7d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:3.0:1787826208.752348:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da70e7a88 (ffff907da5608488/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.752353:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d9102ec30 (ffff907da5608488/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.752360:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d9102ec70 (ffff907da5608488/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.752363:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d9102ecb0 (ffff907da5608488/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.752366:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d9102ecf0 (ffff907da5608488/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.752368:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d9102ed30 (ffff907da5608488/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.752371:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d9102ed70 (ffff907da5608488/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.752373:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d9102edb0 (ffff907da5608488/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.752376:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d9102edf0 (ffff907da5608488/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826208.752413:0:62909:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f2:0x0] ignore/verify layout 0/0, layout version 29 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:3.0:1787826208.752429:0:62909:0:(llite_lib.c:2686:ll_setattr_raw()) COMPLETED file f200b.sanity-flr:[0x200000404:0x1f2:0x0](ffff907d93315b88) current size 9163072, valid attrs 10002168, mode 0, uid -1976721614, gid -1, new size 9163072, atime 1787826208.000000000, mtime 1787826208.000000000, ctime 1787826208.000000000, rc 0 00000080:00200080:3.0:1787826208.752455:0:62909:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88) cmd=400866b1 arg=0 00000080:00200080:3.0:1787826208.752465:0:62909:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88) cmd=400866b1 arg=3 00000080:00200008:3.0:1787826208.752474:0:62909:0:(file.c:6313:ll_getattr_dentry()) START file f200b.sanity-flr:[0x200000404:0x1f2:0x0](ffff907d93315b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787826208.752483:0:62909:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88),name=f200b.sanity-flr 00000002:00010000:3.0:1787826208.756219:0:62909:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f2:0x0]) in obj [0x200000404:0x1f2:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826208.756246:0:62909:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 30 res 8589935620/498 (0 0) 00010000:00010000:3.0:1787826208.756326:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042000/0xe5f736940357b23e lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f2:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 62909 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.756338:0:62909:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042000/0xe5f736940357b23e lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f2:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 62909 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.756350:0:62909:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042000/0xe5f736940357b23e lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f2:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 62909 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826208.756384:0:62909:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83526940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:62909:1874670011330048:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826208.757091:0:62909:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d99937f00 going to sleep for 11 seconds 00000100:00100000:3.0:1787826208.768086:0:62909:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83526940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:62909:1874670011330048:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826208.768124:0:62909:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042000/0xe5f736940357b23e lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1f2:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ea8b expref: -99 pid: 62909 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.768140:0:62909:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042000/0xe5f736940357b23e lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ea8b expref: -99 pid: 62909 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.768155:0:62909:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042000/0xe5f736940357b23e lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ea8b expref: -99 pid: 62909 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.768169:0:62909:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042000/0xe5f736940357b23e lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ea8b expref: -99 pid: 62909 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.768189:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042000/0xe5f736940357b23e lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ea8b expref: -99 pid: 62909 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.768199:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042000/0xe5f736940357b23e lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ea8b expref: -99 pid: 62909 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.768208:0:62909:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042000/0xe5f736940357b23e lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ea8b expref: -99 pid: 62909 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826208.768223:0:62909:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d83526940 x1874670011330048/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/2288 e 0 to 0 dl 1787826224 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1787826208.768236:0:62909:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 1704 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042000/0xe5f736940357b23e lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ea8b expref: -99 pid: 62909 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1787826208.768253:0:62909:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042000/0xe5f736940357b23e lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ea8b expref: -99 pid: 62909 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.768269:0:62909:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/498 (0 0) 00000080:00200000:3.0:1787826208.768278:0:62909:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1f2:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1787826208.768393:0:62909:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1f2:0x0]: layout version change: 29 -> 29 00000080:00010000:3.0:1787826208.768400:0:62909:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f2:0x0](ffff907d93315b88) for lock 0xe5f736940357b23e 00000080:00010000:3.0:1787826208.768408:0:62909:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f2:0x0](ffff907d93315b88) 00000080:00010000:3.0:1787826208.768412:0:62909:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b23e from it ffff9dd1430d7cd8 00010000:00010000:3.0:1787826208.768416:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042000/0xe5f736940357b23e lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0ea8b expref: -99 pid: 62909 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.768432:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042000/0xe5f736940357b23e lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0ea8b expref: -99 pid: 62909 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787826208.768480:0:62909:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f2:0x0] ignore/verify layout 0/1, layout version 29 restore needed 0 00020000:00200000:3.0:1787826208.768565:0:62909:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1f2:0x0]: flr state: 3, move mirror from 1 to 1, have retried: 0, mirror count: 3 00020000:00200000:3.0:1787826208.768570:0:62909:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:3.0:1787826208.768577:0:62909:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1f2:0x0] 00000008:00010000:3.0:1787826208.768604:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826208.768638:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d83faec30, osc lock ffff907d963b9d40, flags 0x400000400001002 00000008:00010000:3.0:1787826208.768651:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826208.768653:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d83faec70, osc lock ffff907d963b9728, flags 0x400000400001002 00000008:00010000:3.0:1787826208.768660:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826208.768662:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d83faecb0, osc lock ffff907d963b8fd8, flags 0x400000400001002 00000008:00010000:3.0:1787826208.768668:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826208.768669:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d83faecf0, osc lock ffff907d963b9e78, flags 0x400000400001002 00000008:00010000:3.0:1787826208.768675:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826208.768676:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d83faed30, osc lock ffff907d963b95f0, flags 0x400000400001002 00000008:00010000:3.0:1787826208.768695:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826208.768698:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d83faed70, osc lock ffff907d963b8270, flags 0x400000400001002 00000008:00010000:3.0:1787826208.768707:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826208.768709:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d83faedb0, osc lock ffff907d963b8000, flags 0x400000400001002 00000008:00010000:3.0:1787826208.768716:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826208.768717:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d83faedf0, osc lock ffff907d963b8138, flags 0x400000400001002 00000008:00010000:3.0:1787826208.768723:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826208.768725:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d83faee30, osc lock ffff907d963b84e0, flags 0x400000400001002 00010000:00010000:3.0:1787826208.768745:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e20400a0/0xe5f736940357b1b9 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x312:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e973 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.768763:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e20400a0/0xe5f736940357b1b9 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x312:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e973 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00000020:00200000:3.0:1787826208.768777:0:62909:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826208.768782:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e20400a0/0xe5f736940357b1b9 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x312:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e973 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.768791:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e20400a0/0xe5f736940357b1b9 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x312:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e973 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.768808:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000862c5d47/0xe5f736940357b1c0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x316:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e96c expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.768818:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 3145727) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000862c5d47/0xe5f736940357b1c0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x316:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e96c expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00000020:00200000:3.0:1787826208.768827:0:62909:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826208.768830:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000862c5d47/0xe5f736940357b1c0 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x316:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e96c expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.768838:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000862c5d47/0xe5f736940357b1c0 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x316:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e96c expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.768856:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000fee745a6/0xe5f736940357b1c7 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x2fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e988 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.768866:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 3145727) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000fee745a6/0xe5f736940357b1c7 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x2fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e988 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00000020:00200000:3.0:1787826208.768881:0:62909:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826208.769063:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000fee745a6/0xe5f736940357b1c7 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x2fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e988 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.769074:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000fee745a6/0xe5f736940357b1c7 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x2fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e988 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.769094:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008e7a0e5a/0xe5f736940357b1ce lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x317:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e97a expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.769103:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (3145728 4194303) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008e7a0e5a/0xe5f736940357b1ce lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x317:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e97a expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00000020:00200000:3.0:1787826208.769112:0:62909:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826208.769115:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008e7a0e5a/0xe5f736940357b1ce lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x317:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e97a expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.769124:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008e7a0e5a/0xe5f736940357b1ce lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x317:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e97a expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.769139:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000007aaddef/0xe5f736940357b1d5 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x2fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e996 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.769149:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (3145728 4194303) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000007aaddef/0xe5f736940357b1d5 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x2fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e996 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00000020:00200000:3.0:1787826208.769159:0:62909:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826208.769162:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000007aaddef/0xe5f736940357b1d5 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x2fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e996 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826208.769170:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000007aaddef/0xe5f736940357b1d5 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x2fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e996 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.769182:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000027595c97/0xe5f736940357b1dc lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x318:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e981 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.769190:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (4194304 16777215) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000027595c97/0xe5f736940357b1dc lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x318:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e981 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00000020:00200000:3.0:1787826208.769197:0:62909:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826208.769200:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000027595c97/0xe5f736940357b1dc lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x318:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e981 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.769209:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000027595c97/0xe5f736940357b1dc lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x318:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e981 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00010000:00010000:3.0:1787826208.769220:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000048d6e517/0xe5f736940357b1e3 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x2fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e99d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.769228:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (4194304 16777215) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000048d6e517/0xe5f736940357b1e3 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x2fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e99d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00000020:00200000:3.0:1787826208.769237:0:62909:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826208.769240:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000048d6e517/0xe5f736940357b1e3 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x2fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e99d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.769249:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000048d6e517/0xe5f736940357b1e3 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x2fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e99d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00010000:00010000:3.0:1787826208.769261:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a04a15b8/0xe5f736940357b1ea lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x301:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e98f expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.769269:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (16777216 18446744073709551615) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a04a15b8/0xe5f736940357b1ea lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x301:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e98f expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00000020:00200000:3.0:1787826208.769277:0:62909:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826208.769279:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a04a15b8/0xe5f736940357b1ea lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x301:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e98f expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.769287:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a04a15b8/0xe5f736940357b1ea lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x301:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e98f expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00010000:00010000:3.0:1787826208.769300:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000099b3a7b7/0xe5f736940357b1f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x31b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e9a4 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.769308:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (16777216 18446744073709551615) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000099b3a7b7/0xe5f736940357b1f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x31b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e9a4 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00000020:00200000:3.0:1787826208.769317:0:62909:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826208.769320:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000099b3a7b7/0xe5f736940357b1f1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x31b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0e9a4 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.769328:0:62909:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000099b3a7b7/0xe5f736940357b1f1 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x31b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0e9a4 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00000020:00200000:3.0:1787826208.769335:0:62909:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1787826208.769348:0:62909:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f2:0x0] updating i_size 9163072 i_blocks 17848 00000020:00010000:3.0:1787826208.769353:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98afe508 (ffff907da5608488/0) at cl_lock_release():245 00000020:00010000:3.0:1787826208.769356:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98afe508 (ffff907da5608488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.769358:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d83faec30 (ffff907da56081b8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.769363:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d83faec70 (ffff907d8662b1b8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.769366:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d83faecb0 (ffff907da5608128/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.769369:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d83faecf0 (ffff907da5608908/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.769372:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d83faed30 (ffff907da5608a28/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.769375:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d83faed70 (ffff907da56085a8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.769378:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d83faedb0 (ffff907da5608368/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.769380:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d83faedf0 (ffff907da5608f38/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.769383:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d83faee30 (ffff907d8662b878/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826208.769386:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98afe508 (ffff907da5608488/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.769388:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d83faec30 (ffff907da5608488/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.769393:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d83faec70 (ffff907da5608488/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.769395:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d83faecb0 (ffff907da5608488/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.769398:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d83faecf0 (ffff907da5608488/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.769402:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d83faed30 (ffff907da5608488/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.769404:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d83faed70 (ffff907da5608488/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.769407:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d83faedb0 (ffff907da5608488/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.769410:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d83faedf0 (ffff907da5608488/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826208.769412:0:62909:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d83faee30 (ffff907da5608488/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826208.769432:0:62909:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f2:0x0] ignore/verify layout 0/1, layout version 29 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:3.0:1787826208.769483:0:62909:0:(llite_lib.c:2427:ll_setattr_raw()) START file f200b.sanity-flr:[0x200000404:0x1f2:0x0](ffff907d93315b88) current size 9163072, valid attrs 2068, mode 0, uid -1976721614, gid -1, new size 9163072, atime 1787826208.000000000, mtime 1787826208.000000000, ctime 1787826208.000000000 00010000:00010000:3.0:1787826208.770672:0:62909:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042000/0xe5f736940357b23e lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0ea8b expref: -99 pid: 62909 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826208.770709:0:62909:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d857ab600/0xe5f736940357b1ab lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f2:0x0].0x0 bits 0x8/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0e957 expref: -99 pid: 62909 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1787826208.770776:0:62909:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042000/0xe5f736940357b23e lrc: 1/0,0 mode: --/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0ea8b expref: -99 pid: 62909 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826208.770783:0:62909:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787826208.770787:0:62909:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042000/0xe5f736940357b23e lrc: 0/0,0 mode: --/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0ea8b expref: -99 pid: 62909 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000020:00100000:3.0:1787826208.770814:0:62909:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1787826208.770829:0:62909:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83526940 x1874670011330304/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826208.770862:0:62909:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83526940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:62909:1874670011330304:192.168.203.104@tcp:36:lfs.0 00000100:00100000:3.0:1787826208.771188:0:62909:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d99937f00 going to sleep for 11 seconds 00000100:00100000:3.0:1787826208.794773:0:62909:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83526940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:62909:1874670011330304:192.168.203.104@tcp:36:lfs.0 00000080:00200000:3.0:1787826208.794835:0:62909:0:(file.c:1848:ll_io_set_mirror()) f200b.sanity-flr: desiginated mirror: 3 00000080:00200000:3.0:1787826208.794844:0:62909:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f2:0x0] ignore/verify layout 0/0, layout version 29 restore needed 0 00020000:00200000:3.0:1787826208.794870:0:62909:0:(lov_io.c:1253:lov_io_iter_init()) component[12] flags 0x11 00020000:00200000:3.0:1787826208.794896:0:62909:0:(lov_io.c:1253:lov_io_iter_init()) component[13] flags 0x10 00000080:00200000:3.0:1787826208.794908:0:62909:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:3.0:1787826208.794943:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787826208.794949:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da6233230, osc lock ffff907d963b84e0, flags 0x0 00000008:00010000:3.0:1787826208.794958:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787826208.794960:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da6233270, osc lock ffff907d963b8138, flags 0x0 00000008:00010000:3.0:1787826208.794963:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787826208.794965:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da62332b0, osc lock ffff907d963b8000, flags 0x0 00000008:00010000:3.0:1787826208.794968:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787826208.794969:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da62332f0, osc lock ffff907d963b8270, flags 0x0 00000008:00010000:3.0:1787826208.794973:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787826208.794974:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da6233330, osc lock ffff907d963b95f0, flags 0x0 00000008:00010000:3.0:1787826208.794977:0:62909:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787826208.794979:0:62909:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da6233370, osc lock ffff907d963b9e78, flags 0x0 00010000:00010000:3.0:1787826208.795056:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a768bf04/0xe5f736940357b245 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.795073:0:62909:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a768bf04/0xe5f736940357b245 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->4194303] (req 0->4194303) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.795085:0:62909:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a768bf04/0xe5f736940357b245 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->4194303] (req 0->4194303) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826208.795108:0:62909:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:62909:1874670011330432:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826208.795193:0:62909:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d99937f00 going to sleep for 11 seconds 00000100:00100000:3.0:1787826208.804104:0:62909:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:62909:1874670011330432:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826208.804147:0:62909:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a768bf04/0xe5f736940357b245 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ea99 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.804177:0:62909:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a768bf04/0xe5f736940357b245 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ea99 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.804192:0:62909:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a768bf04/0xe5f736940357b245 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ea99 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.804212:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a768bf04/0xe5f736940357b245 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ea99 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:3.0:1787826208.805325:0:62909:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 4194304, setting rss=4194304; kms=4194304, end=18446744073709551615 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a768bf04/0xe5f736940357b245 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0ea99 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.805355:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a768bf04/0xe5f736940357b245 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ea99 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.805365:0:62909:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a768bf04/0xe5f736940357b245 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ea99 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.805480:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000002493a11e/0xe5f736940357b24c lrc: 3/0,1 mode: --/PW res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.805490:0:62909:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000002493a11e/0xe5f736940357b24c lrc: 3/0,1 mode: --/PW res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [4194304->8388607] (req 4194304->8388607) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.805502:0:62909:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000002493a11e/0xe5f736940357b24c lrc: 3/0,1 mode: --/PW res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [4194304->8388607] (req 4194304->8388607) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826208.805534:0:62909:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:62909:1874670011330560:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826208.805675:0:62909:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d99937f00 going to sleep for 11 seconds 00000100:00100000:3.0:1787826208.813539:0:62909:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:62909:1874670011330560:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826208.813578:0:62909:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000002493a11e/0xe5f736940357b24c lrc: 4/0,1 mode: --/PW res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0eaa0 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.813784:0:62909:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000002493a11e/0xe5f736940357b24c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0eaa0 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.813803:0:62909:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000002493a11e/0xe5f736940357b24c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0eaa0 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.813824:0:62909:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000002493a11e/0xe5f736940357b24c lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0eaa0 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:3.0:1787826208.813841:0:62909:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 8388608, setting rss=8388608; kms=8388608, end=18446744073709551615 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000002493a11e/0xe5f736940357b24c lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0eaa0 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.813856:0:62909:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000002493a11e/0xe5f736940357b24c lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0eaa0 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.813865:0:62909:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000002493a11e/0xe5f736940357b24c lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0eaa0 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826208.866432:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826208.875477:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826208.882753:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826208.882806:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025453056 00000100:00100000:3.0:1787826208.882819:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025453056 00000100:00100000:3.0:1787826208.882881:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7054 00000100:00100000:3.0:1787826208.882907:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d835243c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:6935:x1874670025453056:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826208.882941:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000862c5d47/0xe5f736940357b1c0 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x316:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0e96c expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.883036:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826208.883044:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d835243c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:6935:x1874670025453056:12345-192.168.203.104@tcp:104: Request processed in 140us (319us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826208.883053:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7054 00010000:00010000:3.0:1787826208.883082:0:60483:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1787826208.883090:0:60483:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000862c5d47/0xe5f736940357b1c0 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x316:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0e96c expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.883101:0:60483:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d857abe00 already unused, calling callback (ffffffffc118e410) 00010000:00010000:3.0:1787826208.883108:0:60483:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000862c5d47/0xe5f736940357b1c0 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x316:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800428400000000 nid: local remote: 0x7918213f4fc0e96c expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00000080:00200000:3.0:1787826208.883135:0:60483:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f2:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787826208.883153:0:60483:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f2:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:3.0:1787826208.883213:0:60483:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000862c5d47/0xe5f736940357b1c0 lrc: 2/0,0 mode: --/PR res: [0x240000400:0x316:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0e96c expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.883223:0:60483:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787826208.884105:0:60483:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000862c5d47/0xe5f736940357b1c0 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x316:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0e96c expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.884116:0:60483:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000862c5d47/0xe5f736940357b1c0 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x316:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0e96c expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.884135:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826208.884137:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826208.884140:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826208.884149:0:2351:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1787826208.884165:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011332352:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1787826208.890560:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025453184 00000100:00100000:3.0:1787826208.890575:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025453184 00000100:00100000:3.0:1787826208.890665:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7055 00000100:00100000:3.0:1787826208.890688:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d9114a580 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:28563:x1874670025453184:12345-192.168.203.104@tcp:105: 00010000:00010000:3.0:1787826208.890717:0:40339:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000875cd77c/0xe5f736940357b276 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x316:0x0].0x0 rrc: 2 type: EXT [1048576->3145727] (req 1048576->3145727) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ead1 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.890740:0:40339:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000875cd77c/0xe5f736940357b276 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x316:0x0].0x0 rrc: 2 type: EXT [1048576->3145727] (req 1048576->3145727) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ead1 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.890752:0:40339:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000875cd77c/0xe5f736940357b276 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x316:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ead1 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.890772:0:40339:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000875cd77c/0xe5f736940357b276 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x316:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x10080000000000 nid: local remote: 0x7918213f4fc0ead1 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.890783:0:40339:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000875cd77c/0xe5f736940357b276 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x316:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x10080000000000 nid: local remote: 0x7918213f4fc0ead1 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.890796:0:40339:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000875cd77c/0xe5f736940357b276 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x316:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x80000000000 nid: local remote: 0x7918213f4fc0ead1 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.890820:0:40339:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff907d90874400) 00000100:00100000:3.0:1787826208.890899:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d9114a580 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:28563:x1874670025453184:12345-192.168.203.104@tcp:105: Request processed in 223us (1147us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826208.890907:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7055 00000100:00100000:3.0:1787826208.893893:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011332352:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1787826208.898647:0:40338:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:3.0:1787826208.898673:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025453312 00000100:00100000:3.0:1787826208.898804:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025453312 00000100:00100000:3.0:1787826208.898911:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7056 00000100:00100000:3.0:1787826208.898933:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907da608c450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:36727:x1874670025453312:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826208.898957:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000fee745a6/0xe5f736940357b1c7 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x2fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0e988 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.899058:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826208.899066:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907da608c450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:36727:x1874670025453312:12345-192.168.203.104@tcp:104: Request processed in 137us (623us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826208.899076:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7056 00000100:00100000:3.0:1787826208.905176:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025453440 00000100:00100000:3.0:1787826208.905188:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025453440 00000100:00100000:3.0:1787826208.911000:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826208.919031:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826208.929663:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025453824 00000100:00100000:3.0:1787826208.929676:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025453824 00000100:00100000:3.0:1787826208.929770:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7060 00000100:00100000:3.0:1787826208.929783:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d9114a940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:36728:x1874670025453824:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826208.929809:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000007aaddef/0xe5f736940357b1d5 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x2fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0e996 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826208.929929:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826208.929937:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d9114a940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:36728:x1874670025453824:12345-192.168.203.104@tcp:104: Request processed in 156us (408us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826208.929946:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7060 00000100:00100000:3.0:1787826208.934543:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025453952 00000100:00100000:3.0:1787826208.934552:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025453952 00000100:00100000:3.0:1787826208.934658:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7061 00000100:00100000:3.0:1787826208.934670:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d9114b840 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:8014:x1874670025453952:12345-192.168.203.104@tcp:105: 00010000:00010000:3.0:1787826208.934696:0:40338:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000009da4f8af/0xe5f736940357b28b lrc: 5/0,1 mode: --/PW res: [0x280000400:0x2fc:0x0].0x0 rrc: 2 type: EXT [3145728->4194303] (req 3145728->4194303) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0eae6 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.934714:0:40338:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000009da4f8af/0xe5f736940357b28b lrc: 5/0,1 mode: --/PW res: [0x280000400:0x2fc:0x0].0x0 rrc: 2 type: EXT [3145728->4194303] (req 3145728->4194303) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0eae6 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.934736:0:40338:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000009da4f8af/0xe5f736940357b28b lrc: 5/0,1 mode: --/PW res: [0x280000400:0x2fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc0eae6 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.934749:0:40338:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000009da4f8af/0xe5f736940357b28b lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x2fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x10080000020000 nid: local remote: 0x7918213f4fc0eae6 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.934761:0:40338:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000009da4f8af/0xe5f736940357b28b lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x2fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x10080000020000 nid: local remote: 0x7918213f4fc0eae6 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.934773:0:40338:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000009da4f8af/0xe5f736940357b28b lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x2fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x80000020000 nid: local remote: 0x7918213f4fc0eae6 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.934804:0:40338:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff907d90874600) 00000100:00100000:3.0:1787826208.934876:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d9114b840 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:8014:x1874670025453952:12345-192.168.203.104@tcp:105: Request processed in 207us (794us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826208.934884:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7061 00000100:00100000:3.0:1787826208.938504:0:50445:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:3.0:1787826208.938530:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025454080 00000100:00100000:3.0:1787826208.938542:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025454080 00000100:00100000:3.0:1787826208.938644:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7062 00000100:00100000:3.0:1787826208.938657:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83facc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:28813:x1874670025454080:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826208.938681:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000027595c97/0xe5f736940357b1dc lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x318:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0e981 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00010000:00010000:3.0:1787826208.938780:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826208.938787:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83facc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:28813:x1874670025454080:12345-192.168.203.104@tcp:104: Request processed in 134us (485us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826208.938796:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7062 00000100:00100000:3.0:1787826208.943418:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025454208 00000100:00100000:3.0:1787826208.943427:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025454208 00000100:00100000:3.0:1787826208.943509:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7063 00000100:00100000:3.0:1787826208.943520:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91148780 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:5932:x1874670025454208:12345-192.168.203.104@tcp:105: 00010000:00010000:3.0:1787826208.943535:0:40339:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006364fa60/0xe5f736940357b292 lrc: 5/0,1 mode: --/PW res: [0x240000400:0x318:0x0].0x0 rrc: 2 type: EXT [4194304->16777215] (req 4194304->16777215) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0eaed expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.943546:0:40339:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006364fa60/0xe5f736940357b292 lrc: 5/0,1 mode: --/PW res: [0x240000400:0x318:0x0].0x0 rrc: 2 type: EXT [4194304->16777215] (req 4194304->16777215) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0eaed expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.943558:0:40339:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006364fa60/0xe5f736940357b292 lrc: 5/0,1 mode: --/PW res: [0x240000400:0x318:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc0eaed expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.943570:0:40339:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006364fa60/0xe5f736940357b292 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x318:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x10080000020000 nid: local remote: 0x7918213f4fc0eaed expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.943581:0:40339:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006364fa60/0xe5f736940357b292 lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x318:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x10080000020000 nid: local remote: 0x7918213f4fc0eaed expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.943593:0:40339:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006364fa60/0xe5f736940357b292 lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x318:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x80000020000 nid: local remote: 0x7918213f4fc0eaed expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.943661:0:40339:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff907d90874c00) 00000100:00100000:3.0:1787826208.944449:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91148780 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:5932:x1874670025454208:12345-192.168.203.104@tcp:105: Request processed in 929us (1467us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826208.944459:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7063 00000100:00100000:3.0:1787826208.948399:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025454336 00000100:00100000:3.0:1787826208.948412:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025454336 00000100:00100000:3.0:1787826208.948494:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7064 00000100:00100000:3.0:1787826208.948508:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d9114a940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:28560:x1874670025454336:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826208.948538:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000048d6e517/0xe5f736940357b1e3 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x2fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0e99d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00010000:00010000:3.0:1787826208.949102:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826208.949109:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d9114a940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:28560:x1874670025454336:12345-192.168.203.104@tcp:104: Request processed in 605us (963us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826208.949119:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7064 00000100:00100000:3.0:1787826208.962253:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826208.971135:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025454592 00000100:00100000:3.0:1787826208.971148:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025454592 00000100:00100000:3.0:1787826208.971213:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7066 00000100:00100000:3.0:1787826208.971331:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d9114b840 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:6932:x1874670025454592:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826208.971358:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a04a15b8/0xe5f736940357b1ea lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x301:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0e98f expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00010000:00010000:3.0:1787826208.971720:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826208.971731:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d9114b840 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:6932:x1874670025454592:12345-192.168.203.104@tcp:104: Request processed in 505us (701us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826208.971740:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7066 00000100:00100000:3.0:1787826208.979064:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025454720 00000100:00100000:3.0:1787826208.979078:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025454720 00000100:00100000:3.0:1787826208.979157:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7067 00000100:00100000:3.0:1787826208.979171:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d9114a580 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:5934:x1874670025454720:12345-192.168.203.104@tcp:105: 00010000:00010000:3.0:1787826208.979196:0:40339:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b2a0 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x301:0x0].0x0 rrc: 2 type: EXT [16777216->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0eafb expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.979212:0:40339:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b2a0 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x301:0x0].0x0 rrc: 2 type: EXT [16777216->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0eafb expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.979235:0:40339:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b2a0 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x301:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc0eafb expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.979250:0:40339:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b2a0 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x301:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x10080000020000 nid: local remote: 0x7918213f4fc0eafb expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.979262:0:40339:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b2a0 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x301:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x10080000020000 nid: local remote: 0x7918213f4fc0eafb expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.979275:0:40339:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b2a0 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x301:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x80000020000 nid: local remote: 0x7918213f4fc0eafb expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.979317:0:40339:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff907d83f0ba00) 00000100:00100000:3.0:1787826208.979399:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d9114a580 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:5934:x1874670025454720:12345-192.168.203.104@tcp:105: Request processed in 229us (741us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826208.979408:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7067 00000100:00100000:3.0:1787826208.984214:0:40338:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:3.0:1787826208.984251:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025454848 00000100:00100000:3.0:1787826208.984262:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025454848 00000100:00100000:3.0:1787826208.984330:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7068 00000100:00100000:3.0:1787826208.984343:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83fadc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:35506:x1874670025454848:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826208.984369:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000099b3a7b7/0xe5f736940357b1f1 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x31b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0e9a4 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 20 lru_type: 1 00010000:00010000:3.0:1787826208.984466:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826208.984473:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83fadc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:35506:x1874670025454848:12345-192.168.203.104@tcp:104: Request processed in 134us (445us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826208.984482:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7068 00000100:00100000:3.0:1787826208.991332:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025454976 00000100:00100000:3.0:1787826208.991343:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025454976 00000100:00100000:3.0:1787826208.991413:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7069 00000100:00100000:3.0:1787826208.991425:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d9114ad00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:5932:x1874670025454976:12345-192.168.203.104@tcp:105: 00010000:00010000:3.0:1787826208.991447:0:50445:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e731006c/0xe5f736940357b2a7 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x31b:0x0].0x0 rrc: 2 type: EXT [16777216->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0eb02 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.991463:0:50445:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e731006c/0xe5f736940357b2a7 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x31b:0x0].0x0 rrc: 2 type: EXT [16777216->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0eb02 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.991474:0:50445:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e731006c/0xe5f736940357b2a7 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x31b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0eb02 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.991487:0:50445:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e731006c/0xe5f736940357b2a7 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x10080000000000 nid: local remote: 0x7918213f4fc0eb02 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.991511:0:50445:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e731006c/0xe5f736940357b2a7 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x31b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x10080000000000 nid: local remote: 0x7918213f4fc0eb02 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.991523:0:50445:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e731006c/0xe5f736940357b2a7 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x31b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x80000000000 nid: local remote: 0x7918213f4fc0eb02 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826208.991546:0:50445:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff907d83f0b800) 00000100:00100000:3.0:1787826208.991672:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d9114ad00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:5932:x1874670025454976:12345-192.168.203.104@tcp:105: Request processed in 248us (521us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826208.991690:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7069 00000100:00100000:3.0:1787826208.994759:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011333888:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1787826209.007085:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025455104 00000100:00100000:3.0:1787826209.007097:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025455104 00000100:00100000:3.0:1787826209.007171:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7070 00000100:00100000:3.0:1787826209.007183:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91148b40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32460:x1874670025455104:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826209.007208:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d857ab600/0xe5f736940357b1ab lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f2:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0e957 expref: -99 pid: 62909 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1787826209.007302:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826209.007309:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91148b40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32460:x1874670025455104:12345-192.168.203.104@tcp:104: Request processed in 128us (350us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826209.007317:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7070 00000080:00200000:3.0:1787826209.246213:0:62918:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826209.250106:0:62918:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826209.250136:0:62918:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d90874a00/0xe5f7369403578b71 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0a4d8 expref: -99 pid: 62013 timeout: 0 lvb_type: 0 lru_score: 119 lru_type: 1 00010000:00010000:3.0:1787826209.250165:0:62918:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d90874a00/0xe5f7369403578b71 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0a4d8 expref: -99 pid: 62013 timeout: 0 lvb_type: 0 lru_score: 119 lru_type: 1 00000080:00200000:3.0:1787826209.250174:0:62918:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826209.250183:0:62918:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826209.250189:0:62918:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f7369403578b71 00000080:00010000:3.0:1787826209.250198:0:62918:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f7369403578b71 from it ffff9dd14310fbf8 00010000:00010000:3.0:1787826209.250203:0:62918:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d90874a00/0xe5f7369403578b71 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0a4d8 expref: -99 pid: 62013 timeout: 0 lvb_type: 0 lru_score: 119 lru_type: 1 00010000:00010000:3.0:1787826209.250212:0:62918:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d90874a00/0xe5f7369403578b71 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0a4d8 expref: -99 pid: 62013 timeout: 0 lvb_type: 0 lru_score: 120 lru_type: 1 00000080:00200000:3.0:1787826209.250247:0:62918:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826209.250273:0:62918:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f200b.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), file ffff907d98b6e480, open_flags c000, mode 0 opened 0 00000080:00200000:3.0:1787826209.250281:0:62918:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f200b.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=open 00000002:00010000:3.0:1787826209.250300:0:62918:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f200b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0140001 00010000:00010000:3.0:1787826209.250368:0:62918:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 62918 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826209.250378:0:62918:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62918 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826209.250390:0:62918:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826209.250406:0:62918:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d835270c0 x1874670011336704/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826209.250423:0:62918:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 62918 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826209.252803:0:62918:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:62918:1874670011336704:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826209.253105:0:62918:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d99937f00 going to sleep for 11 seconds 00000100:00100000:3.0:1787826209.395874:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011336832:192.168.203.104@tcp:3:lfs.0 00000100:00100000:3.0:1787826209.407819:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025458176 00000100:00100000:3.0:1787826209.407830:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025458176 00000100:00100000:3.0:1787826209.407893:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7071 00000100:00100000:3.0:1787826209.407904:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91148f00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:28338:x1874670025458176:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826209.407936:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000045edf04d/0xe5f736940357b206 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x2f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0ea4c expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826209.408027:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826209.408033:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91148f00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:28338:x1874670025458176:12345-192.168.203.104@tcp:104: Request processed in 131us (316us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826209.408040:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7071 00000020:00200000:3.0:1787826209.409291:0:62918:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:3.0:1787826209.409297:0:62918:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 7340032 00000020:00010000:3.0:1787826209.409310:0:62918:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da70e7308 (ffff907da56085a8/0) at cl_lock_release():245 00000020:00010000:3.0:1787826209.409314:0:62918:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da70e7308 (ffff907da56085a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826209.409319:0:62918:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da032efb0 (ffff907d83f04c68/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826209.409323:0:62918:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da70e7308 (ffff907da56085a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826209.409326:0:62918:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da032efb0 (ffff907da56085a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826209.409335:0:62918:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00020000:00200000:3.0:1787826209.409343:0:62918:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 7 chunk: [7340032, 8388608) 67108864, 1048576 00020000:00200000:3.0:1787826209.409348:0:62918:0:(lov_io.c:1253:lov_io_iter_init()) component[3] flags 0x10 00000008:00010000:3.0:1787826209.409438:0:61776:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907d83b790c0 00000008:00100000:3.0:1787826209.409703:0:61776:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d83525a40 - object 0x280000400:762 offset 3145728<>4194304 00000080:00200000:3.0:1787826209.417001:0:62918:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [1792, 2047] 00000008:00010000:3.0:1787826209.417051:0:62918:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:3.0:1787826209.417059:0:62918:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d98a1c4b0, osc lock ffff907d963b8fd8, flags 0x0 00000080:00200000:3.0:1787826209.417084:0:62918:0:(vvp_io.c:874:vvp_io_read_start()) f200b.sanity-flr: read [7340032, 8388608) 00000080:00200000:3.0:1787826209.417106:0:62918:0:(vvp_io.c:194:vvp_prep_size()) [0x200000404:0x1f2:0x0] updating i_size 9163072 00000080:00200000:3.0:1787826209.417115:0:62918:0:(vvp_io.c:932:vvp_io_read_start()) count:1048576 iocb pos:7340032 00000080:00200000:3.0:1787826209.417122:0:62918:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88), size=1048576 (max 335544320), offset=7340032=0x700000, pages 256 (max 81920), lockless, parallel 00000020:00200000:3.0:1787826209.417564:0:62918:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:3.0:1787826209.417569:0:62918:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 8388608 00000020:00010000:3.0:1787826209.417579:0:62918:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da70e7308 (ffff907da56085a8/0) at cl_lock_release():245 00000020:00010000:3.0:1787826209.417583:0:62918:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da70e7308 (ffff907da56085a8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826209.417586:0:62918:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98a1c4b0 (ffff907d83f04488/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826209.417589:0:62918:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da70e7308 (ffff907da56085a8/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826209.417592:0:62918:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98a1c4b0 (ffff907da56085a8/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826209.417604:0:62918:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00020000:00200000:3.0:1787826209.417639:0:62918:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 8 chunk: [8388608, 9437184) 67108864, 1048576 00020000:00200000:3.0:1787826209.417644:0:62918:0:(lov_io.c:1253:lov_io_iter_init()) component[3] flags 0x10 00000008:00010000:3.0:1787826209.417685:0:61776:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907d83b79180 00000008:00100000:3.0:1787826209.417801:0:61776:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d835243c0 - object 0x240000400:789 offset 3145728<>4194304 00010000:00010000:3.0:1787826209.430005:0:62918:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b2c3 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x2f7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826209.430017:0:62918:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b2c3 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x2f7:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826209.430066:0:62918:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000dd58a055/0xe5f736940357b2ca lrc: 3/1,0 mode: --/PR res: [0x240000400:0x313:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826209.431638:0:62918:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000dd58a055/0xe5f736940357b2ca lrc: 3/1,0 mode: --/PR res: [0x240000400:0x313:0x0].0x0 rrc: 2 type: EXT [1048576->2097151] (req 1048576->2097151) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826209.431724:0:62918:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b55f8210/0xe5f736940357b2d1 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x314:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826209.431732:0:62918:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b55f8210/0xe5f736940357b2d1 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x314:0x0].0x0 rrc: 2 type: EXT [1048576->2097151] (req 1048576->2097151) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826209.431762:0:62918:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000aad1631b/0xe5f736940357b2d8 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x2f9:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826209.431781:0:62918:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000aad1631b/0xe5f736940357b2d8 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x2f9:0x0].0x0 rrc: 2 type: EXT [1048576->2097151] (req 1048576->2097151) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826209.431809:0:62918:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000002d53f219/0xe5f736940357b2df lrc: 3/1,0 mode: --/PR res: [0x280000400:0x2fa:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826209.431817:0:62918:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000002d53f219/0xe5f736940357b2df lrc: 3/1,0 mode: --/PR res: [0x280000400:0x2fa:0x0].0x0 rrc: 2 type: EXT [2097152->8388607] (req 2097152->8388607) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826209.431844:0:62918:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c5159d29/0xe5f736940357b2e6 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x315:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826209.431853:0:62918:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c5159d29/0xe5f736940357b2e6 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x315:0x0].0x0 rrc: 2 type: EXT [2097152->8388607] (req 2097152->8388607) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826209.431881:0:62918:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a49bef08/0xe5f736940357b230 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x300:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ea76 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826209.431893:0:62918:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (8388608 18446744073709551615) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a49bef08/0xe5f736940357b230 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x300:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ea76 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:3.0:1787826209.431908:0:62918:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00010000:00010000:3.0:1787826209.431923:0:62918:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a49bef08/0xe5f736940357b230 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x300:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ea76 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826209.431934:0:62918:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a49bef08/0xe5f736940357b230 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x300:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ea76 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826209.431955:0:62918:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000888724c1/0xe5f736940357b237 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x31a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ea7d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826209.431965:0:62918:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (8388608 18446744073709551615) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000888724c1/0xe5f736940357b237 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x31a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ea7d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:3.0:1787826209.431974:0:62918:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00010000:00010000:3.0:1787826209.431977:0:62918:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000888724c1/0xe5f736940357b237 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x31a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ea7d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826209.431985:0:62918:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000888724c1/0xe5f736940357b237 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x31a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ea7d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:3.0:1787826209.431997:0:62918:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000008:00010000:3.0:1787826209.515932:0:2351:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907d83b790c0 00000020:00200000:3.0:1787826209.515947:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:3.0:1787826209.515952:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:3.0:1787826209.515954:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:3.0:1787826209.515955:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:3.0:1787826209.515957:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:3.0:1787826209.515959:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:3.0:1787826209.515961:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:3.0:1787826209.515963:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:3.0:1787826209.515964:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:3.0:1787826209.515965:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:3.0:1787826209.515967:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:3.0:1787826209.515969:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:3.0:1787826209.515970:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:3.0:1787826209.515972:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:3.0:1787826209.515973:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:3.0:1787826209.515975:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:3.0:1787826209.515976:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:3.0:1787826209.515978:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:3.0:1787826209.515980:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:3.0:1787826209.515981:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:3.0:1787826209.515983:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:3.0:1787826209.515984:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:3.0:1787826209.515986:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:3.0:1787826209.515987:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:3.0:1787826209.515988:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:3.0:1787826209.515990:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:3.0:1787826209.515992:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:3.0:1787826209.515994:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:3.0:1787826209.515996:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:3.0:1787826209.515997:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:3.0:1787826209.515999:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:3.0:1787826209.516000:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:3.0:1787826209.516002:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:3.0:1787826209.516003:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:3.0:1787826209.516005:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:3.0:1787826209.516006:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:3.0:1787826209.516008:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:3.0:1787826209.516009:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:3.0:1787826209.516010:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:3.0:1787826209.516012:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:3.0:1787826209.516014:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:3.0:1787826209.516015:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:3.0:1787826209.516016:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:3.0:1787826209.516018:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:3.0:1787826209.516019:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:3.0:1787826209.516021:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:3.0:1787826209.516022:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:3.0:1787826209.516024:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:3.0:1787826209.516026:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:3.0:1787826209.516027:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:3.0:1787826209.516029:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:3.0:1787826209.516030:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:3.0:1787826209.516032:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:3.0:1787826209.516033:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:3.0:1787826209.516035:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:3.0:1787826209.516036:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:3.0:1787826209.516038:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:3.0:1787826209.516039:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:3.0:1787826209.516041:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:3.0:1787826209.516043:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:3.0:1787826209.516045:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:3.0:1787826209.516046:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:3.0:1787826209.516048:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:3.0:1787826209.516049:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:3.0:1787826209.516051:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:3.0:1787826209.516052:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:3.0:1787826209.516054:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:3.0:1787826209.516055:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:3.0:1787826209.516057:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:3.0:1787826209.516058:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:3.0:1787826209.516060:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:3.0:1787826209.516061:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:3.0:1787826209.516063:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:3.0:1787826209.516064:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:3.0:1787826209.516066:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:3.0:1787826209.516067:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:3.0:1787826209.516068:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:3.0:1787826209.516070:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:3.0:1787826209.516071:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:3.0:1787826209.516073:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:3.0:1787826209.516074:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:3.0:1787826209.516076:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:3.0:1787826209.516077:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:3.0:1787826209.516079:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:3.0:1787826209.516080:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:3.0:1787826209.516081:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:3.0:1787826209.516083:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:3.0:1787826209.516084:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:3.0:1787826209.516085:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:3.0:1787826209.516087:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:3.0:1787826209.516088:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:3.0:1787826209.516090:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:3.0:1787826209.516092:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:3.0:1787826209.516094:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:3.0:1787826209.516096:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:3.0:1787826209.516097:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:3.0:1787826209.516099:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:3.0:1787826209.516100:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:3.0:1787826209.516102:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:3.0:1787826209.516103:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:3.0:1787826209.516105:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:3.0:1787826209.516107:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:3.0:1787826209.516109:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:3.0:1787826209.516110:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:3.0:1787826209.516112:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:3.0:1787826209.516114:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:3.0:1787826209.516116:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:3.0:1787826209.516118:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:3.0:1787826209.516119:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:3.0:1787826209.516121:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:3.0:1787826209.516122:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:3.0:1787826209.516124:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:3.0:1787826209.516125:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:3.0:1787826209.516127:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:3.0:1787826209.516129:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:3.0:1787826209.516131:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:3.0:1787826209.516132:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:3.0:1787826209.516134:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:3.0:1787826209.516135:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:3.0:1787826209.516137:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:3.0:1787826209.516138:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:3.0:1787826209.516140:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:3.0:1787826209.516141:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:3.0:1787826209.516144:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:3.0:1787826209.516146:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:3.0:1787826209.516147:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:3.0:1787826209.516149:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:3.0:1787826209.516150:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:3.0:1787826209.516152:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:3.0:1787826209.516153:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:3.0:1787826209.516155:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:3.0:1787826209.516156:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:3.0:1787826209.516158:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:3.0:1787826209.516159:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:3.0:1787826209.516161:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:3.0:1787826209.516163:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:3.0:1787826209.516165:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:3.0:1787826209.516167:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:3.0:1787826209.516169:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:3.0:1787826209.516170:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:3.0:1787826209.516172:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:3.0:1787826209.516174:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:3.0:1787826209.516176:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:3.0:1787826209.516178:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:3.0:1787826209.516179:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:3.0:1787826209.516181:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:3.0:1787826209.516183:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:3.0:1787826209.516184:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:3.0:1787826209.516186:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:3.0:1787826209.516187:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:3.0:1787826209.516189:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:3.0:1787826209.516190:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:3.0:1787826209.516192:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:3.0:1787826209.516194:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:3.0:1787826209.516195:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:3.0:1787826209.516198:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:3.0:1787826209.516199:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:3.0:1787826209.516201:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:3.0:1787826209.516203:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:3.0:1787826209.516204:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:3.0:1787826209.516205:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:3.0:1787826209.516207:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:3.0:1787826209.516209:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:3.0:1787826209.516210:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:3.0:1787826209.516212:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:3.0:1787826209.516214:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:3.0:1787826209.516215:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:3.0:1787826209.516217:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:3.0:1787826209.516219:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:3.0:1787826209.516221:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:3.0:1787826209.516223:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:3.0:1787826209.516224:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:3.0:1787826209.516226:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:3.0:1787826209.516228:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:3.0:1787826209.516230:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:3.0:1787826209.516231:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:3.0:1787826209.516233:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:3.0:1787826209.516234:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:3.0:1787826209.516236:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:3.0:1787826209.516237:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:3.0:1787826209.516238:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:3.0:1787826209.516240:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:3.0:1787826209.516242:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:3.0:1787826209.516243:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:3.0:1787826209.516245:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:3.0:1787826209.516246:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:3.0:1787826209.516248:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:3.0:1787826209.516250:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:3.0:1787826209.516252:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:3.0:1787826209.516253:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:3.0:1787826209.516255:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:3.0:1787826209.516256:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:3.0:1787826209.516258:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:3.0:1787826209.516260:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:3.0:1787826209.516261:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:3.0:1787826209.516262:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:3.0:1787826209.516264:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:3.0:1787826209.516265:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:3.0:1787826209.516267:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:3.0:1787826209.516269:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:3.0:1787826209.516271:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:3.0:1787826209.516273:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:3.0:1787826209.516275:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:3.0:1787826209.516276:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:3.0:1787826209.516278:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:3.0:1787826209.516279:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:3.0:1787826209.516281:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:3.0:1787826209.516282:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:3.0:1787826209.516284:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:3.0:1787826209.516285:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:3.0:1787826209.516287:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:3.0:1787826209.516289:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:3.0:1787826209.516290:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:3.0:1787826209.516292:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:3.0:1787826209.516294:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:3.0:1787826209.516296:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:3.0:1787826209.516297:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:3.0:1787826209.516299:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:3.0:1787826209.516301:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:3.0:1787826209.516303:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:3.0:1787826209.516304:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:3.0:1787826209.516306:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:3.0:1787826209.516308:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:3.0:1787826209.516309:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:3.0:1787826209.516310:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:3.0:1787826209.516312:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:3.0:1787826209.516314:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:3.0:1787826209.516315:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:3.0:1787826209.516317:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:3.0:1787826209.516319:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:3.0:1787826209.516320:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:3.0:1787826209.516323:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:3.0:1787826209.516324:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:3.0:1787826209.516326:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:3.0:1787826209.516328:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:3.0:1787826209.516329:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:3.0:1787826209.516330:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:3.0:1787826209.516332:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:3.0:1787826209.516333:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:3.0:1787826209.516335:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:3.0:1787826209.516336:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:3.0:1787826209.516337:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:3.0:1787826209.516339:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:3.0:1787826209.516341:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:3.0:1787826209.516343:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:3.0:1787826209.516345:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:3.0:1787826209.516346:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:3.0:1787826209.516348:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:3.0:1787826209.516350:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:3.0:1787826209.516352:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:3.0:1787826209.516353:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:3.0:1787826209.516355:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:3.0:1787826209.516357:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:3.0:1787826209.516359:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:3.0:1787826209.516361:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:3.0:1787826209.516363:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:3.0:1787826209.516449:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000008:00010000:3.0:1787826209.516458:0:2351:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907d83b790c0 00000100:00100000:3.0:1787826209.516498:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011337728:192.168.203.104@tcp:3:lfs.0 00000008:00010000:3.0:1787826209.566098:0:2351:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907d83b79180 00000020:00200000:3.0:1787826209.566116:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:3.0:1787826209.566122:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:3.0:1787826209.566124:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:3.0:1787826209.566126:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:3.0:1787826209.566128:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:3.0:1787826209.566131:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:3.0:1787826209.566132:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:3.0:1787826209.566134:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:3.0:1787826209.566136:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:3.0:1787826209.566138:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:3.0:1787826209.566140:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:3.0:1787826209.566142:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:3.0:1787826209.566143:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:3.0:1787826209.566145:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:3.0:1787826209.566147:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:3.0:1787826209.566149:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:3.0:1787826209.566151:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:3.0:1787826209.566153:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:3.0:1787826209.566155:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:3.0:1787826209.566156:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:3.0:1787826209.566158:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:3.0:1787826209.566160:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:3.0:1787826209.566163:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:3.0:1787826209.566165:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:3.0:1787826209.566167:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:3.0:1787826209.566169:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:3.0:1787826209.566171:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:3.0:1787826209.566172:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:3.0:1787826209.566174:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:3.0:1787826209.566176:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:3.0:1787826209.566177:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:3.0:1787826209.566179:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:3.0:1787826209.566180:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:3.0:1787826209.566182:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:3.0:1787826209.566183:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:3.0:1787826209.566185:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:3.0:1787826209.566187:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:3.0:1787826209.566188:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:3.0:1787826209.566190:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:3.0:1787826209.566191:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:3.0:1787826209.566193:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:3.0:1787826209.566194:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:3.0:1787826209.566195:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:3.0:1787826209.566196:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:3.0:1787826209.566198:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:3.0:1787826209.566199:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:3.0:1787826209.566201:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:3.0:1787826209.566202:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:3.0:1787826209.566203:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:3.0:1787826209.566205:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:3.0:1787826209.566206:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:3.0:1787826209.566207:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:3.0:1787826209.566209:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:3.0:1787826209.566210:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:3.0:1787826209.566212:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:3.0:1787826209.566213:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:3.0:1787826209.566214:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:3.0:1787826209.566215:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:3.0:1787826209.566216:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:3.0:1787826209.566218:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:3.0:1787826209.566219:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:3.0:1787826209.566220:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:3.0:1787826209.566221:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:3.0:1787826209.566223:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:3.0:1787826209.566225:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:3.0:1787826209.566227:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:3.0:1787826209.566229:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:3.0:1787826209.566231:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:3.0:1787826209.566233:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:3.0:1787826209.566234:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:3.0:1787826209.566236:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:3.0:1787826209.566238:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:3.0:1787826209.566240:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:3.0:1787826209.566242:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:3.0:1787826209.566244:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:3.0:1787826209.566246:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:3.0:1787826209.566248:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:3.0:1787826209.566250:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:3.0:1787826209.566252:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:3.0:1787826209.566253:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:3.0:1787826209.566255:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:3.0:1787826209.566257:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:3.0:1787826209.566258:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:3.0:1787826209.566259:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:3.0:1787826209.566261:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:3.0:1787826209.566262:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:3.0:1787826209.566265:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:3.0:1787826209.566267:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:3.0:1787826209.566269:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:3.0:1787826209.566270:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:3.0:1787826209.566272:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:3.0:1787826209.566274:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:3.0:1787826209.566276:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:3.0:1787826209.566277:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:3.0:1787826209.566279:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:3.0:1787826209.566280:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:3.0:1787826209.566282:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:3.0:1787826209.566284:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:3.0:1787826209.566286:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:3.0:1787826209.566287:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:3.0:1787826209.566289:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:3.0:1787826209.566291:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:3.0:1787826209.566292:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:3.0:1787826209.566294:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:3.0:1787826209.566296:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:3.0:1787826209.566297:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:3.0:1787826209.566298:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:3.0:1787826209.566300:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:3.0:1787826209.566301:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:3.0:1787826209.566303:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:3.0:1787826209.566304:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:3.0:1787826209.566305:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:3.0:1787826209.566307:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:3.0:1787826209.566309:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:3.0:1787826209.566310:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:3.0:1787826209.566312:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:3.0:1787826209.566314:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:3.0:1787826209.566316:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:3.0:1787826209.566318:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:3.0:1787826209.566319:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:3.0:1787826209.566322:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:3.0:1787826209.566323:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:3.0:1787826209.566325:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:3.0:1787826209.566327:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:3.0:1787826209.566329:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:3.0:1787826209.566330:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:3.0:1787826209.566332:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:3.0:1787826209.566333:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:3.0:1787826209.566335:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:3.0:1787826209.566337:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:3.0:1787826209.566339:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:3.0:1787826209.566341:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:3.0:1787826209.566343:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:3.0:1787826209.566345:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:3.0:1787826209.566347:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:3.0:1787826209.566348:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:3.0:1787826209.566350:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:3.0:1787826209.566352:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:3.0:1787826209.566353:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:3.0:1787826209.566355:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:3.0:1787826209.566357:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:3.0:1787826209.566359:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:3.0:1787826209.566361:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:3.0:1787826209.566362:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:3.0:1787826209.566364:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:3.0:1787826209.566365:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:3.0:1787826209.566367:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:3.0:1787826209.566369:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:3.0:1787826209.566370:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:3.0:1787826209.566372:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:3.0:1787826209.566374:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:3.0:1787826209.566376:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:3.0:1787826209.566377:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:3.0:1787826209.566379:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:3.0:1787826209.566381:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:3.0:1787826209.566382:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:3.0:1787826209.566384:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:3.0:1787826209.566385:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:3.0:1787826209.566387:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:3.0:1787826209.566388:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:3.0:1787826209.566390:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:3.0:1787826209.566392:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:3.0:1787826209.566393:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:3.0:1787826209.566395:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:3.0:1787826209.566397:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:3.0:1787826209.566399:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:3.0:1787826209.566400:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:3.0:1787826209.566402:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:3.0:1787826209.566404:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:3.0:1787826209.566406:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:3.0:1787826209.566408:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:3.0:1787826209.566410:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:3.0:1787826209.566411:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:3.0:1787826209.566413:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:3.0:1787826209.566414:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:3.0:1787826209.566416:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:3.0:1787826209.566417:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:3.0:1787826209.566419:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:3.0:1787826209.566421:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:3.0:1787826209.566423:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:3.0:1787826209.566424:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:3.0:1787826209.566426:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:3.0:1787826209.566428:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:3.0:1787826209.566430:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:3.0:1787826209.566431:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:3.0:1787826209.566433:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:3.0:1787826209.566434:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:3.0:1787826209.566436:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:3.0:1787826209.566438:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:3.0:1787826209.566439:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:3.0:1787826209.566441:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:3.0:1787826209.566443:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:3.0:1787826209.566444:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:3.0:1787826209.566446:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:3.0:1787826209.566447:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:3.0:1787826209.566449:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:3.0:1787826209.566451:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:3.0:1787826209.566453:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:3.0:1787826209.566455:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:3.0:1787826209.566456:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:3.0:1787826209.566458:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:3.0:1787826209.566460:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:3.0:1787826209.566462:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:3.0:1787826209.566463:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:3.0:1787826209.566465:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:3.0:1787826209.566466:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:3.0:1787826209.566468:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:3.0:1787826209.566470:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:3.0:1787826209.566472:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:3.0:1787826209.566473:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:3.0:1787826209.566475:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:3.0:1787826209.566477:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:3.0:1787826209.566478:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:3.0:1787826209.566480:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:3.0:1787826209.566482:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:3.0:1787826209.566484:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:3.0:1787826209.566485:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:3.0:1787826209.566487:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:3.0:1787826209.566488:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:3.0:1787826209.566490:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:3.0:1787826209.566491:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:3.0:1787826209.566493:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:3.0:1787826209.566495:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:3.0:1787826209.566496:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:3.0:1787826209.566498:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:3.0:1787826209.566500:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:3.0:1787826209.566502:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:3.0:1787826209.566503:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:3.0:1787826209.566505:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:3.0:1787826209.566507:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:3.0:1787826209.566509:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:3.0:1787826209.566511:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:3.0:1787826209.566512:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:3.0:1787826209.566514:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:3.0:1787826209.566516:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:3.0:1787826209.566517:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:3.0:1787826209.566519:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:3.0:1787826209.566521:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:3.0:1787826209.566523:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:3.0:1787826209.566525:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:3.0:1787826209.566526:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:3.0:1787826209.566528:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:3.0:1787826209.566530:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:3.0:1787826209.566533:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:3.0:1787826209.566534:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:3.0:1787826209.566536:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:3.0:1787826209.566538:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:3.0:1787826209.566539:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:3.0:1787826209.566541:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:3.0:1787826209.566543:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:3.0:1787826209.566544:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:3.0:1787826209.566546:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:3.0:1787826209.566548:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:3.0:1787826209.566550:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:3.0:1787826209.566552:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:3.0:1787826209.566554:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:3.0:1787826209.566687:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000008:00010000:3.0:1787826209.566696:0:2351:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907d83b79180 00000100:00100000:3.0:1787826209.566730:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835243c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011337856:192.168.203.104@tcp:3:lfs.0 00000080:00200000:3.0:1787826209.580925:0:62918:0:(file.c:2207:ll_file_io_generic()) Range unlock [0, 16383] 00000080:00200000:3.0:1787826209.580956:0:62918:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f2:0x0] ignore/verify layout 0/0, layout version 30 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787826209.580963:0:62918:0:(file.c:2240:ll_file_io_generic()) f200b.sanity-flr: 1 io complete with rc: 0, result: 9163072, restart: 0 00000020:00200000:3.0:1787826209.580967:0:62918:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1787826210.130291:0:62921:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88), inode mode 81a4 mask 4 00000080:00200000:3.0:1787826210.130484:0:62921:0:(xattr.c:760:ll_xattr_get()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88), xattr lov 00000080:00200008:3.0:1787826210.130547:0:62921:0:(file.c:6313:ll_getattr_dentry()) START file f200b.sanity-flr:[0x200000404:0x1f2:0x0](ffff907d93315b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787826210.130556:0:62921:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88),name=f200b.sanity-flr 00000002:00010000:3.0:1787826210.130599:0:62921:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f2:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826210.130651:0:62921:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0eb56 expref: -99 pid: 62918 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1787826210.130668:0:62921:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0eb56 expref: -99 pid: 62918 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1787826210.130677:0:62921:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826210.130686:0:62921:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f2:0x0](ffff907d93315b88) 00000080:00010000:3.0:1787826210.130690:0:62921:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f2:0x0](ffff907d93315b88) for lock 0xe5f736940357b2bc 00000080:00010000:3.0:1787826210.130710:0:62921:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b2bc from it ffff9dd143107cd8 00010000:00010000:3.0:1787826210.130715:0:62921:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0eb56 expref: -99 pid: 62918 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1787826210.130724:0:62921:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0eb56 expref: -99 pid: 62918 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1787826210.235996:0:62921:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [512, 767] 00000008:00010000:3.0:1787826210.236008:0:62921:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:3.0:1787826210.236012:0:62921:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d98a1c4b0, osc lock ffff907d963b8fd8, flags 0x0 00000080:00200000:3.0:1787826210.236018:0:62921:0:(vvp_io.c:874:vvp_io_read_start()) f200b.sanity-flr: read [2097152, 3145728) 00000080:00200000:3.0:1787826210.236026:0:62921:0:(vvp_io.c:194:vvp_prep_size()) [0x200000404:0x1f2:0x0] updating i_size 9163072 00000080:00200000:3.0:1787826210.236031:0:62921:0:(vvp_io.c:932:vvp_io_read_start()) count:1048576 iocb pos:2097152 00000080:00200000:3.0:1787826210.236035:0:62921:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88), size=1048576 (max 335544320), offset=2097152=0x200000, pages 256 (max 81920), lockless, parallel 00000020:00200000:3.0:1787826210.236250:0:62921:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:3.0:1787826210.236253:0:62921:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 3145728 00000020:00010000:3.0:1787826210.236257:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da70e77b8 (ffff907da5608c68/0) at cl_lock_release():245 00000020:00010000:3.0:1787826210.236260:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da70e77b8 (ffff907da5608c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826210.236263:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98a1c4b0 (ffff907d83f04248/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826210.236266:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da70e77b8 (ffff907da5608c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826210.236269:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98a1c4b0 (ffff907da5608c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826210.236276:0:62921:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00020000:00200000:3.0:1787826210.236279:0:62921:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 3 chunk: [3145728, 4194304) 67108864, 1048576 00020000:00200000:3.0:1787826210.236282:0:62921:0:(lov_io.c:1253:lov_io_iter_init()) component[6] flags 0x10 00000080:00200000:3.0:1787826210.236291:0:62921:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [768, 1023] 00000008:00010000:3.0:1787826210.236296:0:62921:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:3.0:1787826210.236299:0:62921:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d98a1c4b0, osc lock ffff907d963b8fd8, flags 0x0 00000080:00200000:3.0:1787826210.236302:0:62921:0:(vvp_io.c:874:vvp_io_read_start()) f200b.sanity-flr: read [3145728, 4194304) 00000080:00200000:3.0:1787826210.236307:0:62921:0:(vvp_io.c:194:vvp_prep_size()) [0x200000404:0x1f2:0x0] updating i_size 9163072 00000080:00200000:3.0:1787826210.236310:0:62921:0:(vvp_io.c:932:vvp_io_read_start()) count:1048576 iocb pos:3145728 00000080:00200000:3.0:1787826210.236313:0:62921:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88), size=1048576 (max 335544320), offset=3145728=0x300000, pages 256 (max 81920), lockless, parallel 00000020:00200000:3.0:1787826210.236526:0:62921:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:3.0:1787826210.236529:0:62921:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 4194304 00000020:00010000:3.0:1787826210.236533:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da70e77b8 (ffff907da5608c68/0) at cl_lock_release():245 00000020:00010000:3.0:1787826210.236536:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da70e77b8 (ffff907da5608c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826210.236539:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98a1c4b0 (ffff907d83f04ab8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826210.236541:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da70e77b8 (ffff907da5608c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826210.236544:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98a1c4b0 (ffff907da5608c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826210.236548:0:62921:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00020000:00200000:3.0:1787826210.236551:0:62921:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 4 chunk: [4194304, 5242880) 67108864, 1048576 00020000:00200000:3.0:1787826210.236554:0:62921:0:(lov_io.c:1253:lov_io_iter_init()) component[6] flags 0x10 00000008:00010000:3.0:1787826210.236575:0:61776:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907d83b790c0 00000008:00100000:3.0:1787826210.236829:0:61776:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d835243c0 - object 0x280000400:763 offset 1048576<>2097152 00000100:00100000:3.0:1787826210.236893:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835243c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011340032:192.168.203.104@tcp:3:lfs.0 00000080:00200000:3.0:1787826210.237578:0:62921:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [1024, 1279] 00000008:00010000:3.0:1787826210.237593:0:62921:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:3.0:1787826210.237596:0:62921:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d98a1c4b0, osc lock ffff907d963b8fd8, flags 0x0 00000080:00200000:3.0:1787826210.237602:0:62921:0:(vvp_io.c:874:vvp_io_read_start()) f200b.sanity-flr: read [4194304, 5242880) 00000080:00200000:3.0:1787826210.237628:0:62921:0:(vvp_io.c:194:vvp_prep_size()) [0x200000404:0x1f2:0x0] updating i_size 9163072 00000080:00200000:3.0:1787826210.237633:0:62921:0:(vvp_io.c:932:vvp_io_read_start()) count:1048576 iocb pos:4194304 00000080:00200000:3.0:1787826210.237638:0:62921:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88), size=1048576 (max 335544320), offset=4194304=0x400000, pages 256 (max 81920), lockless, parallel 00000100:00100000:3.0:1787826210.266232:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7079 00000100:00100000:3.0:1787826210.266245:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91149e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:15253:x1874670025460096:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826210.266257:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000009da4f8af/0xe5f736940357b28b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x2fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800420000020000 nid: local remote: 0x7918213f4fc0eae6 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826210.266308:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826210.266315:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91149e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:15253:x1874670025460096:12345-192.168.203.104@tcp:104: Request processed in 74us (1121us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826210.266321:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7079 00000008:00010000:3.0:1787826210.338305:0:2351:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907d83b790c0 00000020:00200000:3.0:1787826210.338323:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:3.0:1787826210.338328:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:3.0:1787826210.338330:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:3.0:1787826210.338332:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:3.0:1787826210.338333:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:3.0:1787826210.338335:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:3.0:1787826210.338336:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:3.0:1787826210.338338:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:3.0:1787826210.338339:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:3.0:1787826210.338341:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:3.0:1787826210.338342:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:3.0:1787826210.338344:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:3.0:1787826210.338345:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:3.0:1787826210.338347:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:3.0:1787826210.338348:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:3.0:1787826210.338350:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:3.0:1787826210.338351:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:3.0:1787826210.338353:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:3.0:1787826210.338354:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:3.0:1787826210.338355:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:3.0:1787826210.338357:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:3.0:1787826210.338358:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:3.0:1787826210.338360:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:3.0:1787826210.338361:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:3.0:1787826210.338364:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:3.0:1787826210.338365:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:3.0:1787826210.338367:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:3.0:1787826210.338369:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:3.0:1787826210.338371:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:3.0:1787826210.338373:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:3.0:1787826210.338374:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:3.0:1787826210.338376:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:3.0:1787826210.338378:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:3.0:1787826210.338380:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:3.0:1787826210.338382:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:3.0:1787826210.338384:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:3.0:1787826210.338386:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:3.0:1787826210.338388:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:3.0:1787826210.338390:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:3.0:1787826210.338391:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:3.0:1787826210.338393:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:3.0:1787826210.338395:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:3.0:1787826210.338397:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:3.0:1787826210.338399:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:3.0:1787826210.338400:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:3.0:1787826210.338401:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:3.0:1787826210.338403:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:3.0:1787826210.338404:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:3.0:1787826210.338406:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:3.0:1787826210.338407:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:3.0:1787826210.338409:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:3.0:1787826210.338411:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:3.0:1787826210.338412:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:3.0:1787826210.338413:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:3.0:1787826210.338415:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:3.0:1787826210.338417:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:3.0:1787826210.338419:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:3.0:1787826210.338421:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:3.0:1787826210.338423:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:3.0:1787826210.338424:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:3.0:1787826210.338426:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:3.0:1787826210.338428:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:3.0:1787826210.338429:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:3.0:1787826210.338431:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:3.0:1787826210.338433:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:3.0:1787826210.338434:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:3.0:1787826210.338436:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:3.0:1787826210.338437:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:3.0:1787826210.338439:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:3.0:1787826210.338440:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:3.0:1787826210.338442:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:3.0:1787826210.338443:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:3.0:1787826210.338445:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:3.0:1787826210.338447:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:3.0:1787826210.338449:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:3.0:1787826210.338451:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:3.0:1787826210.338452:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:3.0:1787826210.338454:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:3.0:1787826210.338456:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:3.0:1787826210.338458:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:3.0:1787826210.338460:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:3.0:1787826210.338461:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:3.0:1787826210.338463:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:3.0:1787826210.338465:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:3.0:1787826210.338467:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:3.0:1787826210.338468:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:3.0:1787826210.338470:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:3.0:1787826210.338471:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:3.0:1787826210.338474:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:3.0:1787826210.338476:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:3.0:1787826210.338489:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:3.0:1787826210.338492:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:3.0:1787826210.338493:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:3.0:1787826210.338495:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:3.0:1787826210.338497:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:3.0:1787826210.338498:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:3.0:1787826210.338500:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:3.0:1787826210.338502:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:3.0:1787826210.338503:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:3.0:1787826210.338505:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:3.0:1787826210.338507:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:3.0:1787826210.338509:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:3.0:1787826210.338511:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:3.0:1787826210.338513:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:3.0:1787826210.338515:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:3.0:1787826210.338516:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:3.0:1787826210.338518:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:3.0:1787826210.338520:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:3.0:1787826210.338522:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:3.0:1787826210.338524:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:3.0:1787826210.338526:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:3.0:1787826210.338528:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:3.0:1787826210.338530:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:3.0:1787826210.338532:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:3.0:1787826210.338533:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:3.0:1787826210.338534:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:3.0:1787826210.338536:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:3.0:1787826210.338538:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:3.0:1787826210.338539:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:3.0:1787826210.338541:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:3.0:1787826210.338543:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:3.0:1787826210.338545:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:3.0:1787826210.338546:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:3.0:1787826210.338548:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:3.0:1787826210.338550:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:3.0:1787826210.338551:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:3.0:1787826210.338553:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:3.0:1787826210.338554:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:3.0:1787826210.338556:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:3.0:1787826210.338557:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:3.0:1787826210.338559:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:3.0:1787826210.338998:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:3.0:1787826210.339003:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:3.0:1787826210.339005:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:3.0:1787826210.339007:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:3.0:1787826210.339009:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:3.0:1787826210.339010:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:3.0:1787826210.339012:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:3.0:1787826210.339013:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:3.0:1787826210.339014:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:3.0:1787826210.339016:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:3.0:1787826210.339017:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:3.0:1787826210.339019:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:3.0:1787826210.339020:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:3.0:1787826210.339022:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:3.0:1787826210.339023:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:3.0:1787826210.339025:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:3.0:1787826210.339026:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:3.0:1787826210.339028:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:3.0:1787826210.339029:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:3.0:1787826210.339030:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:3.0:1787826210.339032:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:3.0:1787826210.339034:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:3.0:1787826210.339036:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:3.0:1787826210.339038:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:3.0:1787826210.339040:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:3.0:1787826210.339042:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:3.0:1787826210.339043:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:3.0:1787826210.339044:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:3.0:1787826210.339046:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:3.0:1787826210.339047:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:3.0:1787826210.339049:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:3.0:1787826210.339050:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:3.0:1787826210.339052:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:3.0:1787826210.339054:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:3.0:1787826210.339056:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:3.0:1787826210.339057:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:3.0:1787826210.339058:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:3.0:1787826210.339060:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:3.0:1787826210.339062:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:3.0:1787826210.339063:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:3.0:1787826210.339065:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:3.0:1787826210.339066:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:3.0:1787826210.339068:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:3.0:1787826210.339069:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:3.0:1787826210.339071:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:3.0:1787826210.339072:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:3.0:1787826210.339074:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:3.0:1787826210.339075:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:3.0:1787826210.339077:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:3.0:1787826210.339079:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:3.0:1787826210.339081:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:3.0:1787826210.339082:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:3.0:1787826210.339084:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:3.0:1787826210.339086:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:3.0:1787826210.339088:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:3.0:1787826210.339089:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:3.0:1787826210.339091:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:3.0:1787826210.339092:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:3.0:1787826210.339094:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:3.0:1787826210.339095:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:3.0:1787826210.339097:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:3.0:1787826210.339098:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:3.0:1787826210.339099:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:3.0:1787826210.339101:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:3.0:1787826210.339103:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:3.0:1787826210.339104:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:3.0:1787826210.339105:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:3.0:1787826210.339107:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:3.0:1787826210.339108:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:3.0:1787826210.339109:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:3.0:1787826210.339111:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:3.0:1787826210.339113:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:3.0:1787826210.339114:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:3.0:1787826210.339117:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:3.0:1787826210.339118:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:3.0:1787826210.339119:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:3.0:1787826210.339121:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:3.0:1787826210.339122:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:3.0:1787826210.339124:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:3.0:1787826210.339126:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:3.0:1787826210.339127:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:3.0:1787826210.339128:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:3.0:1787826210.339130:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:3.0:1787826210.339131:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:3.0:1787826210.339133:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:3.0:1787826210.339135:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:3.0:1787826210.339137:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:3.0:1787826210.339138:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:3.0:1787826210.339140:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:3.0:1787826210.339141:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:3.0:1787826210.339142:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:3.0:1787826210.339144:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:3.0:1787826210.339145:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:3.0:1787826210.339146:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:3.0:1787826210.339147:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:3.0:1787826210.339149:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:3.0:1787826210.339150:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:3.0:1787826210.339152:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:3.0:1787826210.339154:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:3.0:1787826210.339155:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:3.0:1787826210.339156:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:3.0:1787826210.339158:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:3.0:1787826210.339159:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:3.0:1787826210.339161:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:3.0:1787826210.339162:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:3.0:1787826210.339163:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:3.0:1787826210.339164:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:3.0:1787826210.339166:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:3.0:1787826210.339167:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:3.0:1787826210.339169:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:3.0:1787826210.339170:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:3.0:1787826210.339171:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:3.0:1787826210.339172:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:3.0:1787826210.339173:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:3.0:1787826210.339175:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:3.0:1787826210.339177:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:3.0:1787826210.339179:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:3.0:1787826210.339181:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:3.0:1787826210.339182:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:3.0:1787826210.339184:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:3.0:1787826210.339185:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:3.0:1787826210.339187:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:3.0:1787826210.339188:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:3.0:1787826210.339190:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:3.0:1787826210.339191:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:3.0:1787826210.339294:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000008:00010000:3.0:1787826210.339314:0:2351:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907d83b790c0 00000100:00100000:3.0:1787826210.339356:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835243c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011340032:192.168.203.104@tcp:3:lfs.0 00000080:00200000:3.0:1787826210.668332:0:62921:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f2:0x0] updating i_size 9163072 i_blocks 35728 00000020:00010000:3.0:1787826210.668348:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da61e9b28 (ffff907d8662b248/0) at cl_lock_release():245 00000020:00010000:3.0:1787826210.668353:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da61e9b28 (ffff907d8662b248/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826210.668359:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da6256830 (ffff907d8662ba28/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826210.668367:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da6256870 (ffff907d8662bd88/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826210.668370:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da62568b0 (ffff907d8662b6c8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826210.668373:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da62568f0 (ffff907da5608518/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826210.668378:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da6256930 (ffff907d8662b008/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826210.668382:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da6256970 (ffff907d8662bb48/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826210.668385:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da62569b0 (ffff907d83f04878/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826210.668389:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da62569f0 (ffff907d8662b488/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826210.668393:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da6256a30 (ffff907d8662bab8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826210.668396:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da61e9b28 (ffff907d8662b248/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826210.668400:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da6256830 (ffff907d8662b248/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826210.668407:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da6256870 (ffff907d8662b248/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826210.668410:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da62568b0 (ffff907d8662b248/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826210.668413:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da62568f0 (ffff907d8662b248/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826210.668416:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da6256930 (ffff907d8662b248/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826210.668419:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da6256970 (ffff907d8662b248/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826210.668421:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da62569b0 (ffff907d8662b248/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826210.668424:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da62569f0 (ffff907d8662b248/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826210.668426:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da6256a30 (ffff907d8662b248/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826210.668438:0:62921:0:(vvp_io.c:932:vvp_io_read_start()) count:274112 iocb pos:9163072 00000080:00200000:3.0:1787826210.668444:0:62921:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88), size=274112 (max 335544320), offset=9163072=0x8bd140, pages 67 (max 81920), lockless, parallel, unaligned 00000020:00010000:3.0:1787826210.668462:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b14e8 (ffff907d8662b248/0) at cl_lock_release():245 00000020:00010000:3.0:1787826210.668464:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b14e8 (ffff907d8662b248/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826210.668466:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da032e9b0 (ffff907d8662bb48/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826210.668469:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b14e8 (ffff907d8662b248/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826210.668471:0:62921:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da032e9b0 (ffff907d8662b248/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826210.668479:0:62921:0:(vvp_io.c:533:vvp_io_advance()) advancing 0 bytes 00000020:00200000:3.0:1787826210.668487:0:62921:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1787826210.668491:0:62921:0:(file.c:2207:ll_file_io_generic()) Range unlock [2237, 18621] 00000080:00200000:3.0:1787826210.668785:0:62921:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f2:0x0] ignore/verify layout 0/0, layout version 30 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787826210.668791:0:62921:0:(file.c:2240:ll_file_io_generic()) f200b.sanity-flr: 1 io complete with rc: 0, result: 0, restart: 0 00000020:00200000:3.0:1787826210.668794:0:62921:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1787826210.668829:0:62921:0:(file.c:2326:ll_file_io_generic()) iot: 1, result: 0 00000080:00200080:3.0:1787826210.668850:0:62921:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88) cmd=400866b1 arg=0 00000080:00200080:3.0:1787826210.877434:0:62924:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88) cmd=400866b1 arg=3 00000080:00200008:3.0:1787826210.877463:0:62924:0:(file.c:6313:ll_getattr_dentry()) START file f200b.sanity-flr:[0x200000404:0x1f2:0x0](ffff907d93315b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787826210.877475:0:62924:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88),name=f200b.sanity-flr 00000002:00010000:3.0:1787826210.877517:0:62924:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f2:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826210.877543:0:62924:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0eb56 expref: -99 pid: 62918 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 1 00010000:00010000:3.0:1787826210.877733:0:62924:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0eb56 expref: -99 pid: 62918 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 1 00000080:00200000:3.0:1787826210.877745:0:62924:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826210.877758:0:62924:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f2:0x0](ffff907d93315b88) 00000080:00010000:3.0:1787826210.877762:0:62924:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f2:0x0](ffff907d93315b88) for lock 0xe5f736940357b2bc 00000080:00010000:3.0:1787826210.878100:0:62924:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b2bc from it ffff9dd14308fcd8 00010000:00010000:3.0:1787826210.878107:0:62924:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0eb56 expref: -99 pid: 62918 timeout: 0 lvb_type: 3 lru_score: 8 lru_type: 1 00010000:00010000:3.0:1787826210.878119:0:62924:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0eb56 expref: -99 pid: 62918 timeout: 0 lvb_type: 3 lru_score: 9 lru_type: 1 00000080:00200008:3.0:1787826210.878167:0:62924:0:(file.c:2485:do_file_read_iter()) START file f200b.sanity-flr:[0x200000404:0x1f2:0x0], ppos: 0, count: 67108864 00000080:00200000:3.0:1787826210.878184:0:62924:0:(file.c:2090:ll_file_io_generic()) f200b.sanity-flr: read ppos: 0, bytes: 67108864 00000080:00200000:3.0:1787826210.878197:0:62924:0:(file.c:1848:ll_io_set_mirror()) f200b.sanity-flr: desiginated mirror: 3 00000020:00200000:3.0:1787826210.878208:0:62924:0:(cl_io.c:218:cl_io_rw_init()) header@00000000d9121b18[0x110, 2, [0x200000404:0x1f2:0x0] hash] 00000020:00200000:3.0:1787826210.878211:0:62924:0:(cl_io.c:218:cl_io_rw_init()) io range: 1 [0, 67108864) 0 0 00000080:00200000:3.0:1787826210.878217:0:62924:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f2:0x0] ignore/verify layout 0/0, layout version 30 restore needed 0 00000080:00200000:3.0:1787826210.878240:0:62924:0:(file.c:2170:ll_file_io_generic()) Range lock [0, 16383] 00020000:00200000:3.0:1787826210.878251:0:62924:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 4194304) 67108864, 4194304 00020000:00200000:3.0:1787826210.878255:0:62924:0:(lov_io.c:1253:lov_io_iter_init()) component[10] flags 0x10 00000080:00200000:3.0:1787826210.878276:0:62924:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 1023] 00000008:00010000:3.0:1787826210.878295:0:62924:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:3.0:1787826210.878301:0:62924:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d98a1c730, osc lock ffff907d963b9728, flags 0x0 00000080:00200000:3.0:1787826210.878310:0:62924:0:(vvp_io.c:874:vvp_io_read_start()) f200b.sanity-flr: read [0, 4194304) 00000080:00200000:3.0:1787826210.878323:0:62924:0:(vvp_io.c:194:vvp_prep_size()) [0x200000404:0x1f2:0x0] updating i_size 9163072 00000080:00200000:3.0:1787826210.878332:0:62924:0:(vvp_io.c:932:vvp_io_read_start()) count:4194304 iocb pos:0 00000080:00200000:3.0:1787826210.878338:0:62924:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88), size=4194304 (max 335544320), offset=0=0x0, pages 1024 (max 81920), lockless, parallel 00000080:00200000:3.0:1787826210.889395:0:62924:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [1024, 2047] 00000008:00010000:3.0:1787826210.889423:0:62924:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:3.0:1787826210.889429:0:62924:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d98a1c6b0, osc lock ffff907d963b9d40, flags 0x0 00000080:00200000:3.0:1787826210.889441:0:62924:0:(vvp_io.c:874:vvp_io_read_start()) f200b.sanity-flr: read [4194304, 8388608) 00000080:00200000:3.0:1787826210.889455:0:62924:0:(vvp_io.c:194:vvp_prep_size()) [0x200000404:0x1f2:0x0] updating i_size 9163072 00000080:00200000:3.0:1787826210.889463:0:62924:0:(vvp_io.c:932:vvp_io_read_start()) count:4194304 iocb pos:4194304 00000080:00200000:3.0:1787826210.889471:0:62924:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88), size=4194304 (max 335544320), offset=4194304=0x400000, pages 1024 (max 81920), lockless, parallel 00000100:00100000:3.0:1787826210.897970:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826210.898074:0:40339:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:3.0:1787826210.898092:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025460480 00000100:00100000:3.0:1787826210.898099:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025460480 00000100:00100000:3.0:1787826210.898162:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7082 00000100:00100000:3.0:1787826210.898174:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d88637c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:28339:x1874670025460480:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826210.898193:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a768bf04/0xe5f736940357b245 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0ea99 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826210.898253:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826210.898259:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d88637c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:28339:x1874670025460480:12345-192.168.203.104@tcp:104: Request processed in 87us (293us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826210.898266:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7082 00000100:00100000:3.0:1787826210.900862:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826210.900901:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025460608 00000100:00100000:3.0:1787826210.900920:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025460608 00000100:00100000:3.0:1787826210.900966:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7083 00000100:00100000:3.0:1787826210.900977:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d835243c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:28333:x1874670025460608:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826210.900995:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000002493a11e/0xe5f736940357b24c lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0eaa0 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826210.901070:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:3.0:1787826210.901076:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d835243c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:28333:x1874670025460608:12345-192.168.203.104@tcp:104: Request processed in 102us (224us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826210.901084:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7083 00000100:00100000:3.0:1787826210.902160:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000008:00010000:3.0:1787826211.029586:0:2350:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907da56f8480 00000020:00200000:3.0:1787826211.029602:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1023 items remaining. 00000020:00200000:3.0:1787826211.029656:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1022 items remaining. 00000020:00200000:3.0:1787826211.029658:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1021 items remaining. 00000020:00200000:3.0:1787826211.029660:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1020 items remaining. 00000020:00200000:3.0:1787826211.029662:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1019 items remaining. 00000020:00200000:3.0:1787826211.029664:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1018 items remaining. 00000020:00200000:3.0:1787826211.029665:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1017 items remaining. 00000020:00200000:3.0:1787826211.029667:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1016 items remaining. 00000020:00200000:3.0:1787826211.029669:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1015 items remaining. 00000020:00200000:3.0:1787826211.029671:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1014 items remaining. 00000020:00200000:3.0:1787826211.029673:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1013 items remaining. 00000020:00200000:3.0:1787826211.029674:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1012 items remaining. 00000020:00200000:3.0:1787826211.029676:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1011 items remaining. 00000020:00200000:3.0:1787826211.029678:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1010 items remaining. 00000020:00200000:3.0:1787826211.029680:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1009 items remaining. 00000020:00200000:3.0:1787826211.029682:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1008 items remaining. 00000020:00200000:3.0:1787826211.029684:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1007 items remaining. 00000020:00200000:3.0:1787826211.029686:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1006 items remaining. 00000020:00200000:3.0:1787826211.029688:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1005 items remaining. 00000020:00200000:3.0:1787826211.029689:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1004 items remaining. 00000020:00200000:3.0:1787826211.029691:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1003 items remaining. 00000020:00200000:3.0:1787826211.029693:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1002 items remaining. 00000020:00200000:3.0:1787826211.029695:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1001 items remaining. 00000020:00200000:3.0:1787826211.029697:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1000 items remaining. 00000020:00200000:3.0:1787826211.029699:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 999 items remaining. 00000020:00200000:3.0:1787826211.029701:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 998 items remaining. 00000020:00200000:3.0:1787826211.029704:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 997 items remaining. 00000020:00200000:3.0:1787826211.029707:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 996 items remaining. 00000020:00200000:3.0:1787826211.029709:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 995 items remaining. 00000020:00200000:3.0:1787826211.029710:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 994 items remaining. 00000020:00200000:3.0:1787826211.029712:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 993 items remaining. 00000020:00200000:3.0:1787826211.029714:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 992 items remaining. 00000020:00200000:3.0:1787826211.029716:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 991 items remaining. 00000020:00200000:3.0:1787826211.029718:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 990 items remaining. 00000020:00200000:3.0:1787826211.029719:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 989 items remaining. 00000020:00200000:3.0:1787826211.029721:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 988 items remaining. 00000020:00200000:3.0:1787826211.029722:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 987 items remaining. 00000020:00200000:3.0:1787826211.029723:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 986 items remaining. 00000020:00200000:3.0:1787826211.029725:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 985 items remaining. 00000020:00200000:3.0:1787826211.029727:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 984 items remaining. 00000020:00200000:3.0:1787826211.029728:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 983 items remaining. 00000020:00200000:3.0:1787826211.029730:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 982 items remaining. 00000020:00200000:3.0:1787826211.029732:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 981 items remaining. 00000020:00200000:3.0:1787826211.029733:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 980 items remaining. 00000020:00200000:3.0:1787826211.029735:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 979 items remaining. 00000020:00200000:3.0:1787826211.029736:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 978 items remaining. 00000020:00200000:3.0:1787826211.029738:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 977 items remaining. 00000020:00200000:3.0:1787826211.029740:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 976 items remaining. 00000020:00200000:3.0:1787826211.029741:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 975 items remaining. 00000020:00200000:3.0:1787826211.029743:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 974 items remaining. 00000020:00200000:3.0:1787826211.029745:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 973 items remaining. 00000020:00200000:3.0:1787826211.029746:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 972 items remaining. 00000020:00200000:3.0:1787826211.029748:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 971 items remaining. 00000020:00200000:3.0:1787826211.029750:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 970 items remaining. 00000020:00200000:3.0:1787826211.029751:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 969 items remaining. 00000020:00200000:3.0:1787826211.029753:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 968 items remaining. 00000020:00200000:3.0:1787826211.029754:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 967 items remaining. 00000020:00200000:3.0:1787826211.029756:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 966 items remaining. 00000020:00200000:3.0:1787826211.029757:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 965 items remaining. 00000020:00200000:3.0:1787826211.029760:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 964 items remaining. 00000020:00200000:3.0:1787826211.029762:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 963 items remaining. 00000020:00200000:3.0:1787826211.029764:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 962 items remaining. 00000020:00200000:3.0:1787826211.029765:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 961 items remaining. 00000020:00200000:3.0:1787826211.029766:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 960 items remaining. 00000020:00200000:3.0:1787826211.029768:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 959 items remaining. 00000020:00200000:3.0:1787826211.029770:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 958 items remaining. 00000020:00200000:3.0:1787826211.029771:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 957 items remaining. 00000020:00200000:3.0:1787826211.029773:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 956 items remaining. 00000020:00200000:3.0:1787826211.029774:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 955 items remaining. 00000020:00200000:3.0:1787826211.029776:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 954 items remaining. 00000020:00200000:3.0:1787826211.029778:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 953 items remaining. 00000020:00200000:3.0:1787826211.029779:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 952 items remaining. 00000020:00200000:3.0:1787826211.029781:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 951 items remaining. 00000020:00200000:3.0:1787826211.029782:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 950 items remaining. 00000020:00200000:3.0:1787826211.029784:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 949 items remaining. 00000020:00200000:3.0:1787826211.029786:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 948 items remaining. 00000020:00200000:3.0:1787826211.029787:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 947 items remaining. 00000020:00200000:3.0:1787826211.029789:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 946 items remaining. 00000020:00200000:3.0:1787826211.029791:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 945 items remaining. 00000020:00200000:3.0:1787826211.029793:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 944 items remaining. 00000020:00200000:3.0:1787826211.029794:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 943 items remaining. 00000020:00200000:3.0:1787826211.029796:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 942 items remaining. 00000020:00200000:3.0:1787826211.029798:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 941 items remaining. 00000020:00200000:3.0:1787826211.029799:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 940 items remaining. 00000020:00200000:3.0:1787826211.029801:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 939 items remaining. 00000020:00200000:3.0:1787826211.029802:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 938 items remaining. 00000020:00200000:3.0:1787826211.029804:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 937 items remaining. 00000020:00200000:3.0:1787826211.029805:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 936 items remaining. 00000020:00200000:3.0:1787826211.029807:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 935 items remaining. 00000020:00200000:3.0:1787826211.029808:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 934 items remaining. 00000020:00200000:3.0:1787826211.029809:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 933 items remaining. 00000020:00200000:3.0:1787826211.029812:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 932 items remaining. 00000020:00200000:3.0:1787826211.029814:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 931 items remaining. 00000020:00200000:3.0:1787826211.029816:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 930 items remaining. 00000020:00200000:3.0:1787826211.029818:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 929 items remaining. 00000020:00200000:3.0:1787826211.029819:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 928 items remaining. 00000020:00200000:3.0:1787826211.029821:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 927 items remaining. 00000020:00200000:3.0:1787826211.029823:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 926 items remaining. 00000020:00200000:3.0:1787826211.029825:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 925 items remaining. 00000020:00200000:3.0:1787826211.029826:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 924 items remaining. 00000020:00200000:3.0:1787826211.029828:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 923 items remaining. 00000020:00200000:3.0:1787826211.029830:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 922 items remaining. 00000020:00200000:3.0:1787826211.029831:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 921 items remaining. 00000020:00200000:3.0:1787826211.029833:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 920 items remaining. 00000020:00200000:3.0:1787826211.029835:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 919 items remaining. 00000020:00200000:3.0:1787826211.029837:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 918 items remaining. 00000020:00200000:3.0:1787826211.029839:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 917 items remaining. 00000020:00200000:3.0:1787826211.029841:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 916 items remaining. 00000020:00200000:3.0:1787826211.029843:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 915 items remaining. 00000020:00200000:3.0:1787826211.029845:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 914 items remaining. 00000020:00200000:3.0:1787826211.029847:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 913 items remaining. 00000020:00200000:3.0:1787826211.029849:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 912 items remaining. 00000020:00200000:3.0:1787826211.029850:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 911 items remaining. 00000020:00200000:3.0:1787826211.029852:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 910 items remaining. 00000020:00200000:3.0:1787826211.029854:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 909 items remaining. 00000020:00200000:3.0:1787826211.029857:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 908 items remaining. 00000020:00200000:3.0:1787826211.029858:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 907 items remaining. 00000020:00200000:3.0:1787826211.029860:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 906 items remaining. 00000020:00200000:3.0:1787826211.029862:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 905 items remaining. 00000020:00200000:3.0:1787826211.029864:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 904 items remaining. 00000020:00200000:3.0:1787826211.029866:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 903 items remaining. 00000020:00200000:3.0:1787826211.029867:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 902 items remaining. 00000020:00200000:3.0:1787826211.029869:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 901 items remaining. 00000020:00200000:3.0:1787826211.029871:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 900 items remaining. 00000020:00200000:3.0:1787826211.029873:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 899 items remaining. 00000020:00200000:3.0:1787826211.029875:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 898 items remaining. 00000020:00200000:3.0:1787826211.029877:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 897 items remaining. 00000020:00200000:3.0:1787826211.029879:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 896 items remaining. 00000020:00200000:3.0:1787826211.029880:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 895 items remaining. 00000020:00200000:3.0:1787826211.029882:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 894 items remaining. 00000020:00200000:3.0:1787826211.029884:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 893 items remaining. 00000020:00200000:3.0:1787826211.029886:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 892 items remaining. 00000020:00200000:3.0:1787826211.029887:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 891 items remaining. 00000020:00200000:3.0:1787826211.029889:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 890 items remaining. 00000020:00200000:3.0:1787826211.029891:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 889 items remaining. 00000020:00200000:3.0:1787826211.029893:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 888 items remaining. 00000020:00200000:3.0:1787826211.029895:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 887 items remaining. 00000020:00200000:3.0:1787826211.029897:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 886 items remaining. 00000020:00200000:3.0:1787826211.029900:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 885 items remaining. 00000020:00200000:3.0:1787826211.029940:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 884 items remaining. 00000020:00200000:3.0:1787826211.029943:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 883 items remaining. 00000020:00200000:3.0:1787826211.029945:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 882 items remaining. 00000020:00200000:3.0:1787826211.029946:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 881 items remaining. 00000020:00200000:3.0:1787826211.029948:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 880 items remaining. 00000020:00200000:3.0:1787826211.029949:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 879 items remaining. 00000020:00200000:3.0:1787826211.029951:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 878 items remaining. 00000020:00200000:3.0:1787826211.029953:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 877 items remaining. 00000020:00200000:3.0:1787826211.029955:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 876 items remaining. 00000020:00200000:3.0:1787826211.029956:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 875 items remaining. 00000020:00200000:3.0:1787826211.029958:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 874 items remaining. 00000020:00200000:3.0:1787826211.029959:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 873 items remaining. 00000020:00200000:3.0:1787826211.029961:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 872 items remaining. 00000020:00200000:3.0:1787826211.029963:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 871 items remaining. 00000020:00200000:3.0:1787826211.029965:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 870 items remaining. 00000020:00200000:3.0:1787826211.029967:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 869 items remaining. 00000020:00200000:3.0:1787826211.029969:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 868 items remaining. 00000020:00200000:3.0:1787826211.029971:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 867 items remaining. 00000020:00200000:3.0:1787826211.029973:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 866 items remaining. 00000020:00200000:3.0:1787826211.029975:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 865 items remaining. 00000020:00200000:3.0:1787826211.029977:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 864 items remaining. 00000020:00200000:3.0:1787826211.029979:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 863 items remaining. 00000020:00200000:3.0:1787826211.029980:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 862 items remaining. 00000020:00200000:3.0:1787826211.029982:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 861 items remaining. 00000020:00200000:3.0:1787826211.029983:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 860 items remaining. 00000020:00200000:3.0:1787826211.029985:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 859 items remaining. 00000020:00200000:3.0:1787826211.029987:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 858 items remaining. 00000020:00200000:3.0:1787826211.029989:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 857 items remaining. 00000020:00200000:3.0:1787826211.029991:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 856 items remaining. 00000020:00200000:3.0:1787826211.029993:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 855 items remaining. 00000020:00200000:3.0:1787826211.029995:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 854 items remaining. 00000020:00200000:3.0:1787826211.029997:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 853 items remaining. 00000020:00200000:3.0:1787826211.029999:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 852 items remaining. 00000020:00200000:3.0:1787826211.030001:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 851 items remaining. 00000020:00200000:3.0:1787826211.030003:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 850 items remaining. 00000020:00200000:3.0:1787826211.030005:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 849 items remaining. 00000020:00200000:3.0:1787826211.030007:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 848 items remaining. 00000020:00200000:3.0:1787826211.030008:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 847 items remaining. 00000020:00200000:3.0:1787826211.030010:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 846 items remaining. 00000020:00200000:3.0:1787826211.030012:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 845 items remaining. 00000020:00200000:3.0:1787826211.030014:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 844 items remaining. 00000020:00200000:3.0:1787826211.030016:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 843 items remaining. 00000020:00200000:3.0:1787826211.030017:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 842 items remaining. 00000020:00200000:3.0:1787826211.030019:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 841 items remaining. 00000020:00200000:3.0:1787826211.030020:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 840 items remaining. 00000020:00200000:3.0:1787826211.030022:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 839 items remaining. 00000020:00200000:3.0:1787826211.030024:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 838 items remaining. 00000020:00200000:3.0:1787826211.030025:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 837 items remaining. 00000020:00200000:3.0:1787826211.030028:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 836 items remaining. 00000020:00200000:3.0:1787826211.030029:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 835 items remaining. 00000020:00200000:3.0:1787826211.030031:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 834 items remaining. 00000020:00200000:3.0:1787826211.030034:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 833 items remaining. 00000020:00200000:3.0:1787826211.030036:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 832 items remaining. 00000020:00200000:3.0:1787826211.030037:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 831 items remaining. 00000020:00200000:3.0:1787826211.030039:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 830 items remaining. 00000020:00200000:3.0:1787826211.030041:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 829 items remaining. 00000020:00200000:3.0:1787826211.030043:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 828 items remaining. 00000020:00200000:3.0:1787826211.030044:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 827 items remaining. 00000020:00200000:3.0:1787826211.030046:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 826 items remaining. 00000020:00200000:3.0:1787826211.030048:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 825 items remaining. 00000020:00200000:3.0:1787826211.030050:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 824 items remaining. 00000020:00200000:3.0:1787826211.030052:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 823 items remaining. 00000020:00200000:3.0:1787826211.030053:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 822 items remaining. 00000020:00200000:3.0:1787826211.030055:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 821 items remaining. 00000020:00200000:3.0:1787826211.030057:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 820 items remaining. 00000020:00200000:3.0:1787826211.030059:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 819 items remaining. 00000020:00200000:3.0:1787826211.030061:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 818 items remaining. 00000020:00200000:3.0:1787826211.030063:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 817 items remaining. 00000020:00200000:3.0:1787826211.030065:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 816 items remaining. 00000020:00200000:3.0:1787826211.030067:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 815 items remaining. 00000020:00200000:3.0:1787826211.030069:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 814 items remaining. 00000020:00200000:3.0:1787826211.030071:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 813 items remaining. 00000020:00200000:3.0:1787826211.030073:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 812 items remaining. 00000020:00200000:3.0:1787826211.030074:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 811 items remaining. 00000020:00200000:3.0:1787826211.030076:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 810 items remaining. 00000020:00200000:3.0:1787826211.030077:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 809 items remaining. 00000020:00200000:3.0:1787826211.030079:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 808 items remaining. 00000020:00200000:3.0:1787826211.030081:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 807 items remaining. 00000020:00200000:3.0:1787826211.030082:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 806 items remaining. 00000020:00200000:3.0:1787826211.030084:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 805 items remaining. 00000020:00200000:3.0:1787826211.030086:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 804 items remaining. 00000020:00200000:3.0:1787826211.030088:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 803 items remaining. 00000020:00200000:3.0:1787826211.030090:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 802 items remaining. 00000020:00200000:3.0:1787826211.030092:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 801 items remaining. 00000020:00200000:3.0:1787826211.030093:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 800 items remaining. 00000020:00200000:3.0:1787826211.030095:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 799 items remaining. 00000020:00200000:3.0:1787826211.030096:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 798 items remaining. 00000020:00200000:3.0:1787826211.030098:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 797 items remaining. 00000020:00200000:3.0:1787826211.030099:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 796 items remaining. 00000020:00200000:3.0:1787826211.030102:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 795 items remaining. 00000020:00200000:3.0:1787826211.030103:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 794 items remaining. 00000020:00200000:3.0:1787826211.030105:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 793 items remaining. 00000020:00200000:3.0:1787826211.030107:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 792 items remaining. 00000020:00200000:3.0:1787826211.030109:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 791 items remaining. 00000020:00200000:3.0:1787826211.030111:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 790 items remaining. 00000020:00200000:3.0:1787826211.030113:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 789 items remaining. 00000020:00200000:3.0:1787826211.030115:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 788 items remaining. 00000020:00200000:3.0:1787826211.030117:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 787 items remaining. 00000020:00200000:3.0:1787826211.030119:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 786 items remaining. 00000020:00200000:3.0:1787826211.030121:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 785 items remaining. 00000020:00200000:3.0:1787826211.030123:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 784 items remaining. 00000020:00200000:3.0:1787826211.030124:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 783 items remaining. 00000020:00200000:3.0:1787826211.030126:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 782 items remaining. 00000020:00200000:3.0:1787826211.030128:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 781 items remaining. 00000020:00200000:3.0:1787826211.030129:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 780 items remaining. 00000020:00200000:3.0:1787826211.030131:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 779 items remaining. 00000020:00200000:3.0:1787826211.030132:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 778 items remaining. 00000020:00200000:3.0:1787826211.030134:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 777 items remaining. 00000020:00200000:3.0:1787826211.030136:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 776 items remaining. 00000020:00200000:3.0:1787826211.030138:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 775 items remaining. 00000020:00200000:3.0:1787826211.030140:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 774 items remaining. 00000020:00200000:3.0:1787826211.030142:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 773 items remaining. 00000020:00200000:3.0:1787826211.030145:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 772 items remaining. 00000020:00200000:3.0:1787826211.030146:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 771 items remaining. 00000020:00200000:3.0:1787826211.030148:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 770 items remaining. 00000020:00200000:3.0:1787826211.030150:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 769 items remaining. 00000020:00200000:3.0:1787826211.030152:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 768 items remaining. 00000020:00200000:3.0:1787826211.030153:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 767 items remaining. 00000020:00200000:3.0:1787826211.030155:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 766 items remaining. 00000020:00200000:3.0:1787826211.030167:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 765 items remaining. 00000020:00200000:3.0:1787826211.030169:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 764 items remaining. 00000020:00200000:3.0:1787826211.030171:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 763 items remaining. 00000020:00200000:3.0:1787826211.030173:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 762 items remaining. 00000020:00200000:3.0:1787826211.030174:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 761 items remaining. 00000020:00200000:3.0:1787826211.030176:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 760 items remaining. 00000020:00200000:3.0:1787826211.030178:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 759 items remaining. 00000020:00200000:3.0:1787826211.030180:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 758 items remaining. 00000020:00200000:3.0:1787826211.030182:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 757 items remaining. 00000020:00200000:3.0:1787826211.030184:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 756 items remaining. 00000020:00200000:3.0:1787826211.030186:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 755 items remaining. 00000020:00200000:3.0:1787826211.030188:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 754 items remaining. 00000020:00200000:3.0:1787826211.030190:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 753 items remaining. 00000020:00200000:3.0:1787826211.030192:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 752 items remaining. 00000020:00200000:3.0:1787826211.030194:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 751 items remaining. 00000020:00200000:3.0:1787826211.030196:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 750 items remaining. 00000020:00200000:3.0:1787826211.030198:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 749 items remaining. 00000020:00200000:3.0:1787826211.030200:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 748 items remaining. 00000020:00200000:3.0:1787826211.030202:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 747 items remaining. 00000020:00200000:3.0:1787826211.030204:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 746 items remaining. 00000020:00200000:3.0:1787826211.030206:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 745 items remaining. 00000020:00200000:3.0:1787826211.030207:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 744 items remaining. 00000020:00200000:3.0:1787826211.030209:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 743 items remaining. 00000020:00200000:3.0:1787826211.030211:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 742 items remaining. 00000020:00200000:3.0:1787826211.030213:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 741 items remaining. 00000020:00200000:3.0:1787826211.030215:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 740 items remaining. 00000020:00200000:3.0:1787826211.030217:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 739 items remaining. 00000020:00200000:3.0:1787826211.030219:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 738 items remaining. 00000020:00200000:3.0:1787826211.030221:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 737 items remaining. 00000020:00200000:3.0:1787826211.030223:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 736 items remaining. 00000020:00200000:3.0:1787826211.030225:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 735 items remaining. 00000020:00200000:3.0:1787826211.030227:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 734 items remaining. 00000020:00200000:3.0:1787826211.030229:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 733 items remaining. 00000020:00200000:3.0:1787826211.030230:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 732 items remaining. 00000020:00200000:3.0:1787826211.030232:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 731 items remaining. 00000020:00200000:3.0:1787826211.030234:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 730 items remaining. 00000020:00200000:3.0:1787826211.030236:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 729 items remaining. 00000020:00200000:3.0:1787826211.030238:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 728 items remaining. 00000020:00200000:3.0:1787826211.030240:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 727 items remaining. 00000020:00200000:3.0:1787826211.030241:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 726 items remaining. 00000020:00200000:3.0:1787826211.030243:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 725 items remaining. 00000020:00200000:3.0:1787826211.030245:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 724 items remaining. 00000020:00200000:3.0:1787826211.030247:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 723 items remaining. 00000020:00200000:3.0:1787826211.030250:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 722 items remaining. 00000020:00200000:3.0:1787826211.030261:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 721 items remaining. 00000020:00200000:3.0:1787826211.030263:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 720 items remaining. 00000020:00200000:3.0:1787826211.030265:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 719 items remaining. 00000020:00200000:3.0:1787826211.030267:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 718 items remaining. 00000020:00200000:3.0:1787826211.030269:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 717 items remaining. 00000020:00200000:3.0:1787826211.030271:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 716 items remaining. 00000020:00200000:3.0:1787826211.030273:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 715 items remaining. 00000020:00200000:3.0:1787826211.030275:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 714 items remaining. 00000020:00200000:3.0:1787826211.030277:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 713 items remaining. 00000020:00200000:3.0:1787826211.030279:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 712 items remaining. 00000020:00200000:3.0:1787826211.030289:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 711 items remaining. 00000020:00200000:3.0:1787826211.030291:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 710 items remaining. 00000020:00200000:3.0:1787826211.030293:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 709 items remaining. 00000020:00200000:3.0:1787826211.030295:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 708 items remaining. 00000020:00200000:3.0:1787826211.030298:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 707 items remaining. 00000020:00200000:3.0:1787826211.030299:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 706 items remaining. 00000020:00200000:3.0:1787826211.030301:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 705 items remaining. 00000020:00200000:3.0:1787826211.030303:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 704 items remaining. 00000020:00200000:3.0:1787826211.030304:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 703 items remaining. 00000020:00200000:3.0:1787826211.030306:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 702 items remaining. 00000020:00200000:3.0:1787826211.030308:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 701 items remaining. 00000020:00200000:3.0:1787826211.030309:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 700 items remaining. 00000020:00200000:3.0:1787826211.030311:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 699 items remaining. 00000020:00200000:3.0:1787826211.030313:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 698 items remaining. 00000020:00200000:3.0:1787826211.030315:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 697 items remaining. 00000020:00200000:3.0:1787826211.030317:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 696 items remaining. 00000020:00200000:3.0:1787826211.030319:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 695 items remaining. 00000020:00200000:3.0:1787826211.030322:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 694 items remaining. 00000020:00200000:3.0:1787826211.030324:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 693 items remaining. 00000020:00200000:3.0:1787826211.030326:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 692 items remaining. 00000020:00200000:3.0:1787826211.030327:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 691 items remaining. 00000020:00200000:3.0:1787826211.030329:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 690 items remaining. 00000020:00200000:3.0:1787826211.030331:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 689 items remaining. 00000020:00200000:3.0:1787826211.030333:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 688 items remaining. 00000020:00200000:3.0:1787826211.030335:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 687 items remaining. 00000020:00200000:3.0:1787826211.030337:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 686 items remaining. 00000020:00200000:3.0:1787826211.030339:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 685 items remaining. 00000020:00200000:3.0:1787826211.030341:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 684 items remaining. 00000020:00200000:3.0:1787826211.030342:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 683 items remaining. 00000020:00200000:3.0:1787826211.030345:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 682 items remaining. 00000020:00200000:3.0:1787826211.030346:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 681 items remaining. 00000020:00200000:3.0:1787826211.030348:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 680 items remaining. 00000020:00200000:3.0:1787826211.030350:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 679 items remaining. 00000020:00200000:3.0:1787826211.030352:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 678 items remaining. 00000020:00200000:3.0:1787826211.030353:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 677 items remaining. 00000020:00200000:3.0:1787826211.030356:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 676 items remaining. 00000020:00200000:3.0:1787826211.030357:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 675 items remaining. 00000020:00200000:3.0:1787826211.030359:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 674 items remaining. 00000020:00200000:3.0:1787826211.030361:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 673 items remaining. 00000020:00200000:3.0:1787826211.030363:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 672 items remaining. 00000020:00200000:3.0:1787826211.030365:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 671 items remaining. 00000020:00200000:3.0:1787826211.030366:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 670 items remaining. 00000020:00200000:3.0:1787826211.030368:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 669 items remaining. 00000020:00200000:3.0:1787826211.030369:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 668 items remaining. 00000020:00200000:3.0:1787826211.030371:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 667 items remaining. 00000020:00200000:3.0:1787826211.030372:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 666 items remaining. 00000020:00200000:3.0:1787826211.030655:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 665 items remaining. 00000020:00200000:3.0:1787826211.030658:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 664 items remaining. 00000020:00200000:3.0:1787826211.030661:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 663 items remaining. 00000020:00200000:3.0:1787826211.030663:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 662 items remaining. 00000020:00200000:3.0:1787826211.030664:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 661 items remaining. 00000020:00200000:3.0:1787826211.030666:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 660 items remaining. 00000020:00200000:3.0:1787826211.030668:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 659 items remaining. 00000020:00200000:3.0:1787826211.030670:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 658 items remaining. 00000020:00200000:3.0:1787826211.030672:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 657 items remaining. 00000020:00200000:3.0:1787826211.030674:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 656 items remaining. 00000020:00200000:3.0:1787826211.030676:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 655 items remaining. 00000020:00200000:3.0:1787826211.030678:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 654 items remaining. 00000020:00200000:3.0:1787826211.030680:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 653 items remaining. 00000020:00200000:3.0:1787826211.030682:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 652 items remaining. 00000020:00200000:3.0:1787826211.030684:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 651 items remaining. 00000020:00200000:3.0:1787826211.030686:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 650 items remaining. 00000020:00200000:3.0:1787826211.030688:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 649 items remaining. 00000020:00200000:3.0:1787826211.030690:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 648 items remaining. 00000020:00200000:3.0:1787826211.030691:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 647 items remaining. 00000020:00200000:3.0:1787826211.030693:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 646 items remaining. 00000020:00200000:3.0:1787826211.030694:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 645 items remaining. 00000020:00200000:3.0:1787826211.030697:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 644 items remaining. 00000020:00200000:3.0:1787826211.030698:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 643 items remaining. 00000020:00200000:3.0:1787826211.030700:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 642 items remaining. 00000020:00200000:3.0:1787826211.030702:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 641 items remaining. 00000020:00200000:3.0:1787826211.030704:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 640 items remaining. 00000020:00200000:3.0:1787826211.030706:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 639 items remaining. 00000020:00200000:3.0:1787826211.030707:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 638 items remaining. 00000020:00200000:3.0:1787826211.030709:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 637 items remaining. 00000020:00200000:3.0:1787826211.030711:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 636 items remaining. 00000020:00200000:3.0:1787826211.030712:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 635 items remaining. 00000020:00200000:3.0:1787826211.030713:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 634 items remaining. 00000020:00200000:3.0:1787826211.030716:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 633 items remaining. 00000020:00200000:3.0:1787826211.030717:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 632 items remaining. 00000020:00200000:3.0:1787826211.030719:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 631 items remaining. 00000020:00200000:3.0:1787826211.030721:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 630 items remaining. 00000020:00200000:3.0:1787826211.030723:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 629 items remaining. 00000020:00200000:3.0:1787826211.030725:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 628 items remaining. 00000020:00200000:3.0:1787826211.030726:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 627 items remaining. 00000020:00200000:3.0:1787826211.030728:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 626 items remaining. 00000020:00200000:3.0:1787826211.030730:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 625 items remaining. 00000020:00200000:3.0:1787826211.030731:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 624 items remaining. 00000020:00200000:3.0:1787826211.030733:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 623 items remaining. 00000020:00200000:3.0:1787826211.030735:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 622 items remaining. 00000020:00200000:3.0:1787826211.030737:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 621 items remaining. 00000020:00200000:3.0:1787826211.030739:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 620 items remaining. 00000020:00200000:3.0:1787826211.030740:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 619 items remaining. 00000020:00200000:3.0:1787826211.030742:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 618 items remaining. 00000020:00200000:3.0:1787826211.030744:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 617 items remaining. 00000020:00200000:3.0:1787826211.030745:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 616 items remaining. 00000020:00200000:3.0:1787826211.030747:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 615 items remaining. 00000020:00200000:3.0:1787826211.030748:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 614 items remaining. 00000020:00200000:3.0:1787826211.030750:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 613 items remaining. 00000020:00200000:3.0:1787826211.030752:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 612 items remaining. 00000020:00200000:3.0:1787826211.030753:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 611 items remaining. 00000020:00200000:3.0:1787826211.030755:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 610 items remaining. 00000020:00200000:3.0:1787826211.030757:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 609 items remaining. 00000020:00200000:3.0:1787826211.030758:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 608 items remaining. 00000020:00200000:3.0:1787826211.030760:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 607 items remaining. 00000020:00200000:3.0:1787826211.030762:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 606 items remaining. 00000020:00200000:3.0:1787826211.030763:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 605 items remaining. 00000020:00200000:3.0:1787826211.030765:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 604 items remaining. 00000020:00200000:3.0:1787826211.030767:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 603 items remaining. 00000020:00200000:3.0:1787826211.030769:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 602 items remaining. 00000020:00200000:3.0:1787826211.030771:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 601 items remaining. 00000020:00200000:3.0:1787826211.030772:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 600 items remaining. 00000020:00200000:3.0:1787826211.030774:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 599 items remaining. 00000020:00200000:3.0:1787826211.030776:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 598 items remaining. 00000020:00200000:3.0:1787826211.030777:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 597 items remaining. 00000020:00200000:3.0:1787826211.030779:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 596 items remaining. 00000020:00200000:3.0:1787826211.030781:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 595 items remaining. 00000020:00200000:3.0:1787826211.030783:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 594 items remaining. 00000020:00200000:3.0:1787826211.030785:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 593 items remaining. 00000020:00200000:3.0:1787826211.030787:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 592 items remaining. 00000020:00200000:3.0:1787826211.030789:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 591 items remaining. 00000020:00200000:3.0:1787826211.030791:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 590 items remaining. 00000020:00200000:3.0:1787826211.030792:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 589 items remaining. 00000020:00200000:3.0:1787826211.030795:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 588 items remaining. 00000020:00200000:3.0:1787826211.030796:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 587 items remaining. 00000020:00200000:3.0:1787826211.030798:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 586 items remaining. 00000020:00200000:3.0:1787826211.030800:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 585 items remaining. 00000020:00200000:3.0:1787826211.030801:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 584 items remaining. 00000020:00200000:3.0:1787826211.030803:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 583 items remaining. 00000020:00200000:3.0:1787826211.030805:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 582 items remaining. 00000020:00200000:3.0:1787826211.030807:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 581 items remaining. 00000020:00200000:3.0:1787826211.030809:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 580 items remaining. 00000020:00200000:3.0:1787826211.030811:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 579 items remaining. 00000020:00200000:3.0:1787826211.030813:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 578 items remaining. 00000020:00200000:3.0:1787826211.030815:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 577 items remaining. 00000020:00200000:3.0:1787826211.030817:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 576 items remaining. 00000020:00200000:3.0:1787826211.030819:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 575 items remaining. 00000020:00200000:3.0:1787826211.030821:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 574 items remaining. 00000020:00200000:3.0:1787826211.030823:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 573 items remaining. 00000020:00200000:3.0:1787826211.030825:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 572 items remaining. 00000020:00200000:3.0:1787826211.030826:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 571 items remaining. 00000020:00200000:3.0:1787826211.030829:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 570 items remaining. 00000020:00200000:3.0:1787826211.030831:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 569 items remaining. 00000020:00200000:3.0:1787826211.030832:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 568 items remaining. 00000020:00200000:3.0:1787826211.030834:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 567 items remaining. 00000020:00200000:3.0:1787826211.030836:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 566 items remaining. 00000020:00200000:3.0:1787826211.030837:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 565 items remaining. 00000020:00200000:3.0:1787826211.030839:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 564 items remaining. 00000020:00200000:3.0:1787826211.030841:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 563 items remaining. 00000020:00200000:3.0:1787826211.030842:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 562 items remaining. 00000020:00200000:3.0:1787826211.030844:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 561 items remaining. 00000020:00200000:3.0:1787826211.030846:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 560 items remaining. 00000020:00200000:3.0:1787826211.030847:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 559 items remaining. 00000020:00200000:3.0:1787826211.030849:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 558 items remaining. 00000020:00200000:3.0:1787826211.030850:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 557 items remaining. 00000020:00200000:3.0:1787826211.030852:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 556 items remaining. 00000020:00200000:3.0:1787826211.030854:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 555 items remaining. 00000020:00200000:3.0:1787826211.030856:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 554 items remaining. 00000020:00200000:3.0:1787826211.030857:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 553 items remaining. 00000020:00200000:3.0:1787826211.030859:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 552 items remaining. 00000020:00200000:3.0:1787826211.030860:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 551 items remaining. 00000020:00200000:3.0:1787826211.030862:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 550 items remaining. 00000020:00200000:3.0:1787826211.030864:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 549 items remaining. 00000020:00200000:3.0:1787826211.030866:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 548 items remaining. 00000020:00200000:3.0:1787826211.030867:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 547 items remaining. 00000020:00200000:3.0:1787826211.030869:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 546 items remaining. 00000020:00200000:3.0:1787826211.030871:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 545 items remaining. 00000020:00200000:3.0:1787826211.030873:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 544 items remaining. 00000020:00200000:3.0:1787826211.030874:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 543 items remaining. 00000020:00200000:3.0:1787826211.030876:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 542 items remaining. 00000020:00200000:3.0:1787826211.030878:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 541 items remaining. 00000020:00200000:3.0:1787826211.030879:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 540 items remaining. 00000020:00200000:3.0:1787826211.030881:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 539 items remaining. 00000020:00200000:3.0:1787826211.030883:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 538 items remaining. 00000020:00200000:3.0:1787826211.030884:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 537 items remaining. 00000020:00200000:3.0:1787826211.030886:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 536 items remaining. 00000020:00200000:3.0:1787826211.030888:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 535 items remaining. 00000020:00200000:3.0:1787826211.030890:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 534 items remaining. 00000020:00200000:3.0:1787826211.030893:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 533 items remaining. 00000020:00200000:3.0:1787826211.030895:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 532 items remaining. 00000020:00200000:3.0:1787826211.030897:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 531 items remaining. 00000020:00200000:3.0:1787826211.030898:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 530 items remaining. 00000020:00200000:3.0:1787826211.030900:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 529 items remaining. 00000020:00200000:3.0:1787826211.030911:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 528 items remaining. 00000020:00200000:3.0:1787826211.030913:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 527 items remaining. 00000020:00200000:3.0:1787826211.030915:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 526 items remaining. 00000020:00200000:3.0:1787826211.030916:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 525 items remaining. 00000020:00200000:3.0:1787826211.030918:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 524 items remaining. 00000020:00200000:3.0:1787826211.030920:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 523 items remaining. 00000020:00200000:3.0:1787826211.030921:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 522 items remaining. 00000020:00200000:3.0:1787826211.030922:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 521 items remaining. 00000020:00200000:3.0:1787826211.030924:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 520 items remaining. 00000020:00200000:3.0:1787826211.030926:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 519 items remaining. 00000020:00200000:3.0:1787826211.030927:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 518 items remaining. 00000020:00200000:3.0:1787826211.030929:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 517 items remaining. 00000020:00200000:3.0:1787826211.030932:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 516 items remaining. 00000020:00200000:3.0:1787826211.030934:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 515 items remaining. 00000020:00200000:3.0:1787826211.030936:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 514 items remaining. 00000020:00200000:3.0:1787826211.030938:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 513 items remaining. 00000020:00200000:3.0:1787826211.030939:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 512 items remaining. 00000020:00200000:3.0:1787826211.030941:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 511 items remaining. 00000020:00200000:3.0:1787826211.030942:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 510 items remaining. 00000020:00200000:3.0:1787826211.030944:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 509 items remaining. 00000020:00200000:3.0:1787826211.030945:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 508 items remaining. 00000020:00200000:3.0:1787826211.030947:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 507 items remaining. 00000020:00200000:3.0:1787826211.030948:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 506 items remaining. 00000020:00200000:3.0:1787826211.030950:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 505 items remaining. 00000020:00200000:3.0:1787826211.030951:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 504 items remaining. 00000020:00200000:3.0:1787826211.030953:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 503 items remaining. 00000020:00200000:3.0:1787826211.030955:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 502 items remaining. 00000020:00200000:3.0:1787826211.030957:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 501 items remaining. 00000020:00200000:3.0:1787826211.030958:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 500 items remaining. 00000020:00200000:3.0:1787826211.030960:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 499 items remaining. 00000020:00200000:3.0:1787826211.030961:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 498 items remaining. 00000020:00200000:3.0:1787826211.030963:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 497 items remaining. 00000020:00200000:3.0:1787826211.030964:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 496 items remaining. 00000020:00200000:3.0:1787826211.030966:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 495 items remaining. 00000020:00200000:3.0:1787826211.030968:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 494 items remaining. 00000020:00200000:3.0:1787826211.030969:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 493 items remaining. 00000020:00200000:3.0:1787826211.030971:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 492 items remaining. 00000020:00200000:3.0:1787826211.030972:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 491 items remaining. 00000020:00200000:3.0:1787826211.030974:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 490 items remaining. 00000020:00200000:3.0:1787826211.030975:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 489 items remaining. 00000020:00200000:3.0:1787826211.030977:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 488 items remaining. 00000020:00200000:3.0:1787826211.030979:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 487 items remaining. 00000020:00200000:3.0:1787826211.030980:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 486 items remaining. 00000020:00200000:3.0:1787826211.030982:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 485 items remaining. 00000020:00200000:3.0:1787826211.030984:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 484 items remaining. 00000020:00200000:3.0:1787826211.030986:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 483 items remaining. 00000020:00200000:3.0:1787826211.030988:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 482 items remaining. 00000020:00200000:3.0:1787826211.030990:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 481 items remaining. 00000020:00200000:3.0:1787826211.030991:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 480 items remaining. 00000020:00200000:3.0:1787826211.030993:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 479 items remaining. 00000020:00200000:3.0:1787826211.030995:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 478 items remaining. 00000020:00200000:3.0:1787826211.030996:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 477 items remaining. 00000020:00200000:3.0:1787826211.030998:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 476 items remaining. 00000020:00200000:3.0:1787826211.031000:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 475 items remaining. 00000020:00200000:3.0:1787826211.031002:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 474 items remaining. 00000020:00200000:3.0:1787826211.031003:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 473 items remaining. 00000020:00200000:3.0:1787826211.031006:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 472 items remaining. 00000020:00200000:3.0:1787826211.031007:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 471 items remaining. 00000020:00200000:3.0:1787826211.031009:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 470 items remaining. 00000020:00200000:3.0:1787826211.031011:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 469 items remaining. 00000020:00200000:3.0:1787826211.031013:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 468 items remaining. 00000020:00200000:3.0:1787826211.031015:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 467 items remaining. 00000020:00200000:3.0:1787826211.031016:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 466 items remaining. 00000020:00200000:3.0:1787826211.031018:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 465 items remaining. 00000020:00200000:3.0:1787826211.031020:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 464 items remaining. 00000020:00200000:3.0:1787826211.031022:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 463 items remaining. 00000020:00200000:3.0:1787826211.031024:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 462 items remaining. 00000020:00200000:3.0:1787826211.031026:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 461 items remaining. 00000020:00200000:3.0:1787826211.031028:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 460 items remaining. 00000020:00200000:3.0:1787826211.031030:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 459 items remaining. 00000020:00200000:3.0:1787826211.031032:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 458 items remaining. 00000020:00200000:3.0:1787826211.031034:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 457 items remaining. 00000020:00200000:3.0:1787826211.031035:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 456 items remaining. 00000020:00200000:3.0:1787826211.031038:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 455 items remaining. 00000020:00200000:3.0:1787826211.031040:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 454 items remaining. 00000020:00200000:3.0:1787826211.031041:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 453 items remaining. 00000020:00200000:3.0:1787826211.031043:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 452 items remaining. 00000020:00200000:3.0:1787826211.031045:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 451 items remaining. 00000020:00200000:3.0:1787826211.031047:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 450 items remaining. 00000020:00200000:3.0:1787826211.031048:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 449 items remaining. 00000020:00200000:3.0:1787826211.031050:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 448 items remaining. 00000020:00200000:3.0:1787826211.031052:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 447 items remaining. 00000020:00200000:3.0:1787826211.031054:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 446 items remaining. 00000020:00200000:3.0:1787826211.031056:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 445 items remaining. 00000020:00200000:3.0:1787826211.031057:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 444 items remaining. 00000020:00200000:3.0:1787826211.031058:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 443 items remaining. 00000020:00200000:3.0:1787826211.031060:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 442 items remaining. 00000020:00200000:3.0:1787826211.031061:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 441 items remaining. 00000020:00200000:3.0:1787826211.031063:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 440 items remaining. 00000020:00200000:3.0:1787826211.031065:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 439 items remaining. 00000020:00200000:3.0:1787826211.031066:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 438 items remaining. 00000020:00200000:3.0:1787826211.031068:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 437 items remaining. 00000020:00200000:3.0:1787826211.031070:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 436 items remaining. 00000020:00200000:3.0:1787826211.031071:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 435 items remaining. 00000020:00200000:3.0:1787826211.031073:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 434 items remaining. 00000020:00200000:3.0:1787826211.031075:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 433 items remaining. 00000020:00200000:3.0:1787826211.031076:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 432 items remaining. 00000020:00200000:3.0:1787826211.031078:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 431 items remaining. 00000020:00200000:3.0:1787826211.031080:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 430 items remaining. 00000020:00200000:3.0:1787826211.031082:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 429 items remaining. 00000020:00200000:3.0:1787826211.031084:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 428 items remaining. 00000020:00200000:3.0:1787826211.031086:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 427 items remaining. 00000020:00200000:3.0:1787826211.031087:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 426 items remaining. 00000020:00200000:3.0:1787826211.031089:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 425 items remaining. 00000020:00200000:3.0:1787826211.031091:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 424 items remaining. 00000020:00200000:3.0:1787826211.031092:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 423 items remaining. 00000020:00200000:3.0:1787826211.031094:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 422 items remaining. 00000020:00200000:3.0:1787826211.031096:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 421 items remaining. 00000020:00200000:3.0:1787826211.031097:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 420 items remaining. 00000020:00200000:3.0:1787826211.031099:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 419 items remaining. 00000020:00200000:3.0:1787826211.031102:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 418 items remaining. 00000020:00200000:3.0:1787826211.031104:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 417 items remaining. 00000020:00200000:3.0:1787826211.031105:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 416 items remaining. 00000020:00200000:3.0:1787826211.031107:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 415 items remaining. 00000020:00200000:3.0:1787826211.031109:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 414 items remaining. 00000020:00200000:3.0:1787826211.031110:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 413 items remaining. 00000020:00200000:3.0:1787826211.031112:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 412 items remaining. 00000020:00200000:3.0:1787826211.031114:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 411 items remaining. 00000020:00200000:3.0:1787826211.031115:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 410 items remaining. 00000020:00200000:3.0:1787826211.031117:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 409 items remaining. 00000020:00200000:3.0:1787826211.031119:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 408 items remaining. 00000020:00200000:3.0:1787826211.031121:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 407 items remaining. 00000020:00200000:3.0:1787826211.031123:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 406 items remaining. 00000020:00200000:3.0:1787826211.031125:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 405 items remaining. 00000020:00200000:3.0:1787826211.031126:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 404 items remaining. 00000020:00200000:3.0:1787826211.031128:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 403 items remaining. 00000020:00200000:3.0:1787826211.031130:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 402 items remaining. 00000020:00200000:3.0:1787826211.031132:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 401 items remaining. 00000020:00200000:3.0:1787826211.031134:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 400 items remaining. 00000020:00200000:3.0:1787826211.031136:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 399 items remaining. 00000020:00200000:3.0:1787826211.031137:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 398 items remaining. 00000020:00200000:3.0:1787826211.031139:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 397 items remaining. 00000020:00200000:3.0:1787826211.031141:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 396 items remaining. 00000020:00200000:3.0:1787826211.031143:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 395 items remaining. 00000020:00200000:3.0:1787826211.031144:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 394 items remaining. 00000020:00200000:3.0:1787826211.031146:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 393 items remaining. 00000020:00200000:3.0:1787826211.031148:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 392 items remaining. 00000020:00200000:3.0:1787826211.031150:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 391 items remaining. 00000020:00200000:3.0:1787826211.031152:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 390 items remaining. 00000020:00200000:3.0:1787826211.031164:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 389 items remaining. 00000020:00200000:3.0:1787826211.031167:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 388 items remaining. 00000020:00200000:3.0:1787826211.031169:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 387 items remaining. 00000020:00200000:3.0:1787826211.031170:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 386 items remaining. 00000020:00200000:3.0:1787826211.031172:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 385 items remaining. 00000020:00200000:3.0:1787826211.031174:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 384 items remaining. 00000020:00200000:3.0:1787826211.031175:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 383 items remaining. 00000020:00200000:3.0:1787826211.031177:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 382 items remaining. 00000020:00200000:3.0:1787826211.031179:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 381 items remaining. 00000020:00200000:3.0:1787826211.031181:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 380 items remaining. 00000020:00200000:3.0:1787826211.031182:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 379 items remaining. 00000020:00200000:3.0:1787826211.031184:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 378 items remaining. 00000020:00200000:3.0:1787826211.031186:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 377 items remaining. 00000020:00200000:3.0:1787826211.031187:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 376 items remaining. 00000020:00200000:3.0:1787826211.031189:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 375 items remaining. 00000020:00200000:3.0:1787826211.031191:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 374 items remaining. 00000020:00200000:3.0:1787826211.031193:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 373 items remaining. 00000020:00200000:3.0:1787826211.031195:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 372 items remaining. 00000020:00200000:3.0:1787826211.031196:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 371 items remaining. 00000020:00200000:3.0:1787826211.031198:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 370 items remaining. 00000020:00200000:3.0:1787826211.031200:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 369 items remaining. 00000020:00200000:3.0:1787826211.031202:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 368 items remaining. 00000020:00200000:3.0:1787826211.031203:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 367 items remaining. 00000020:00200000:3.0:1787826211.031205:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 366 items remaining. 00000020:00200000:3.0:1787826211.031207:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 365 items remaining. 00000020:00200000:3.0:1787826211.031209:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 364 items remaining. 00000020:00200000:3.0:1787826211.031211:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 363 items remaining. 00000020:00200000:3.0:1787826211.031212:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 362 items remaining. 00000020:00200000:3.0:1787826211.031214:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 361 items remaining. 00000020:00200000:3.0:1787826211.031215:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 360 items remaining. 00000020:00200000:3.0:1787826211.031217:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 359 items remaining. 00000020:00200000:3.0:1787826211.031219:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 358 items remaining. 00000020:00200000:3.0:1787826211.031221:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 357 items remaining. 00000020:00200000:3.0:1787826211.031224:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 356 items remaining. 00000020:00200000:3.0:1787826211.031226:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 355 items remaining. 00000020:00200000:3.0:1787826211.031227:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 354 items remaining. 00000020:00200000:3.0:1787826211.031229:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 353 items remaining. 00000020:00200000:3.0:1787826211.031231:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 352 items remaining. 00000020:00200000:3.0:1787826211.031233:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 351 items remaining. 00000020:00200000:3.0:1787826211.031235:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 350 items remaining. 00000020:00200000:3.0:1787826211.031236:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 349 items remaining. 00000020:00200000:3.0:1787826211.031238:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 348 items remaining. 00000020:00200000:3.0:1787826211.031240:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 347 items remaining. 00000020:00200000:3.0:1787826211.031241:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 346 items remaining. 00000020:00200000:3.0:1787826211.031243:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 345 items remaining. 00000020:00200000:3.0:1787826211.031245:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 344 items remaining. 00000020:00200000:3.0:1787826211.031246:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 343 items remaining. 00000020:00200000:3.0:1787826211.031248:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 342 items remaining. 00000020:00200000:3.0:1787826211.031250:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 341 items remaining. 00000020:00200000:3.0:1787826211.031252:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 340 items remaining. 00000020:00200000:3.0:1787826211.031254:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 339 items remaining. 00000020:00200000:3.0:1787826211.031256:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 338 items remaining. 00000020:00200000:3.0:1787826211.031258:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 337 items remaining. 00000020:00200000:3.0:1787826211.031260:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 336 items remaining. 00000020:00200000:3.0:1787826211.031262:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 335 items remaining. 00000020:00200000:3.0:1787826211.031264:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 334 items remaining. 00000020:00200000:3.0:1787826211.031265:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 333 items remaining. 00000020:00200000:3.0:1787826211.031268:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 332 items remaining. 00000020:00200000:3.0:1787826211.031269:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 331 items remaining. 00000020:00200000:3.0:1787826211.031271:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 330 items remaining. 00000020:00200000:3.0:1787826211.031273:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 329 items remaining. 00000020:00200000:3.0:1787826211.031275:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 328 items remaining. 00000020:00200000:3.0:1787826211.031277:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 327 items remaining. 00000020:00200000:3.0:1787826211.031279:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 326 items remaining. 00000020:00200000:3.0:1787826211.031444:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 325 items remaining. 00000020:00200000:3.0:1787826211.031448:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 324 items remaining. 00000020:00200000:3.0:1787826211.031450:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 323 items remaining. 00000020:00200000:3.0:1787826211.031452:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 322 items remaining. 00000020:00200000:3.0:1787826211.031454:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 321 items remaining. 00000020:00200000:3.0:1787826211.031455:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 320 items remaining. 00000020:00200000:3.0:1787826211.031457:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 319 items remaining. 00000020:00200000:3.0:1787826211.031459:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 318 items remaining. 00000020:00200000:3.0:1787826211.031460:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 317 items remaining. 00000020:00200000:3.0:1787826211.031462:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 316 items remaining. 00000020:00200000:3.0:1787826211.031464:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 315 items remaining. 00000020:00200000:3.0:1787826211.031465:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 314 items remaining. 00000020:00200000:3.0:1787826211.031467:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 313 items remaining. 00000020:00200000:3.0:1787826211.031469:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 312 items remaining. 00000020:00200000:3.0:1787826211.031471:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 311 items remaining. 00000020:00200000:3.0:1787826211.031473:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 310 items remaining. 00000020:00200000:3.0:1787826211.031475:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 309 items remaining. 00000020:00200000:3.0:1787826211.031478:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 308 items remaining. 00000020:00200000:3.0:1787826211.031480:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 307 items remaining. 00000020:00200000:3.0:1787826211.031482:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 306 items remaining. 00000020:00200000:3.0:1787826211.031484:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 305 items remaining. 00000020:00200000:3.0:1787826211.031486:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 304 items remaining. 00000020:00200000:3.0:1787826211.031488:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 303 items remaining. 00000020:00200000:3.0:1787826211.031490:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 302 items remaining. 00000020:00200000:3.0:1787826211.031492:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 301 items remaining. 00000020:00200000:3.0:1787826211.031494:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 300 items remaining. 00000020:00200000:3.0:1787826211.031496:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 299 items remaining. 00000020:00200000:3.0:1787826211.031498:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 298 items remaining. 00000020:00200000:3.0:1787826211.031500:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 297 items remaining. 00000020:00200000:3.0:1787826211.031501:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 296 items remaining. 00000020:00200000:3.0:1787826211.031503:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 295 items remaining. 00000020:00200000:3.0:1787826211.031505:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 294 items remaining. 00000020:00200000:3.0:1787826211.031506:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 293 items remaining. 00000020:00200000:3.0:1787826211.031509:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 292 items remaining. 00000020:00200000:3.0:1787826211.031511:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 291 items remaining. 00000020:00200000:3.0:1787826211.031513:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 290 items remaining. 00000020:00200000:3.0:1787826211.031515:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 289 items remaining. 00000020:00200000:3.0:1787826211.031517:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 288 items remaining. 00000020:00200000:3.0:1787826211.031519:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 287 items remaining. 00000020:00200000:3.0:1787826211.031520:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 286 items remaining. 00000020:00200000:3.0:1787826211.031522:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 285 items remaining. 00000020:00200000:3.0:1787826211.031523:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 284 items remaining. 00000020:00200000:3.0:1787826211.031526:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 283 items remaining. 00000020:00200000:3.0:1787826211.031528:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 282 items remaining. 00000020:00200000:3.0:1787826211.031530:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 281 items remaining. 00000020:00200000:3.0:1787826211.031532:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 280 items remaining. 00000020:00200000:3.0:1787826211.031534:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 279 items remaining. 00000020:00200000:3.0:1787826211.031536:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 278 items remaining. 00000020:00200000:3.0:1787826211.031538:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 277 items remaining. 00000020:00200000:3.0:1787826211.031540:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 276 items remaining. 00000020:00200000:3.0:1787826211.031542:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 275 items remaining. 00000020:00200000:3.0:1787826211.031544:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 274 items remaining. 00000020:00200000:3.0:1787826211.031546:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 273 items remaining. 00000020:00200000:3.0:1787826211.031548:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 272 items remaining. 00000020:00200000:3.0:1787826211.031550:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 271 items remaining. 00000020:00200000:3.0:1787826211.031553:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 270 items remaining. 00000020:00200000:3.0:1787826211.031555:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 269 items remaining. 00000020:00200000:3.0:1787826211.031557:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 268 items remaining. 00000020:00200000:3.0:1787826211.031558:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 267 items remaining. 00000020:00200000:3.0:1787826211.031560:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 266 items remaining. 00000020:00200000:3.0:1787826211.031562:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 265 items remaining. 00000020:00200000:3.0:1787826211.031564:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 264 items remaining. 00000020:00200000:3.0:1787826211.031566:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 263 items remaining. 00000020:00200000:3.0:1787826211.031568:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 262 items remaining. 00000020:00200000:3.0:1787826211.031570:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 261 items remaining. 00000020:00200000:3.0:1787826211.031572:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 260 items remaining. 00000020:00200000:3.0:1787826211.031574:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 259 items remaining. 00000020:00200000:3.0:1787826211.031576:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 258 items remaining. 00000020:00200000:3.0:1787826211.031578:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 257 items remaining. 00000020:00200000:3.0:1787826211.031579:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000020:00200000:3.0:1787826211.031581:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:3.0:1787826211.031582:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:3.0:1787826211.031584:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:3.0:1787826211.031585:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:3.0:1787826211.031587:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:3.0:1787826211.031588:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:3.0:1787826211.031590:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:3.0:1787826211.031592:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:3.0:1787826211.031593:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:3.0:1787826211.031595:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:3.0:1787826211.031597:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:3.0:1787826211.031599:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:3.0:1787826211.031601:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:3.0:1787826211.031603:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:3.0:1787826211.031652:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:3.0:1787826211.031656:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:3.0:1787826211.031658:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:3.0:1787826211.031660:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:3.0:1787826211.031662:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:3.0:1787826211.031664:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:3.0:1787826211.031666:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:3.0:1787826211.031668:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:3.0:1787826211.031669:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:3.0:1787826211.031671:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:3.0:1787826211.031673:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:3.0:1787826211.031674:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:3.0:1787826211.031676:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:3.0:1787826211.031678:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:3.0:1787826211.031680:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:3.0:1787826211.031682:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:3.0:1787826211.031685:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:3.0:1787826211.031687:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:3.0:1787826211.031689:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:3.0:1787826211.031691:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:3.0:1787826211.031694:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:3.0:1787826211.031696:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:3.0:1787826211.031698:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:3.0:1787826211.031700:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:3.0:1787826211.031703:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:3.0:1787826211.031705:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:3.0:1787826211.031708:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:3.0:1787826211.031710:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:3.0:1787826211.031712:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:3.0:1787826211.031714:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:3.0:1787826211.031717:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:3.0:1787826211.031719:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:3.0:1787826211.031721:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:3.0:1787826211.031723:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:3.0:1787826211.031725:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:3.0:1787826211.031728:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:3.0:1787826211.031730:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:3.0:1787826211.031732:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:3.0:1787826211.031734:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:3.0:1787826211.031736:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:3.0:1787826211.031738:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:3.0:1787826211.031740:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:3.0:1787826211.031742:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:3.0:1787826211.031743:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:3.0:1787826211.031746:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:3.0:1787826211.031748:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:3.0:1787826211.031750:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:3.0:1787826211.031752:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:3.0:1787826211.031754:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:3.0:1787826211.031755:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:3.0:1787826211.031757:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:3.0:1787826211.031758:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:3.0:1787826211.031760:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:3.0:1787826211.031762:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:3.0:1787826211.031763:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:3.0:1787826211.031765:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:3.0:1787826211.031767:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:3.0:1787826211.031769:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:3.0:1787826211.031771:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:3.0:1787826211.031773:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:3.0:1787826211.031775:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:3.0:1787826211.031777:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:3.0:1787826211.031779:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:3.0:1787826211.031780:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:3.0:1787826211.031782:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:3.0:1787826211.031784:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:3.0:1787826211.031786:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:3.0:1787826211.031788:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:3.0:1787826211.031791:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:3.0:1787826211.031793:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:3.0:1787826211.031795:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:3.0:1787826211.031796:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:3.0:1787826211.031798:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:3.0:1787826211.031800:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:3.0:1787826211.031802:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:3.0:1787826211.031804:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:3.0:1787826211.031806:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:3.0:1787826211.031808:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:3.0:1787826211.031811:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:3.0:1787826211.031813:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:3.0:1787826211.031815:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:3.0:1787826211.031817:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:3.0:1787826211.031818:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:3.0:1787826211.031820:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:3.0:1787826211.031822:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:3.0:1787826211.031824:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:3.0:1787826211.031826:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:3.0:1787826211.031827:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:3.0:1787826211.031829:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:3.0:1787826211.031831:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:3.0:1787826211.031833:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:3.0:1787826211.031835:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:3.0:1787826211.031849:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:3.0:1787826211.031851:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:3.0:1787826211.031853:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:3.0:1787826211.031855:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:3.0:1787826211.031857:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:3.0:1787826211.031859:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:3.0:1787826211.031861:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:3.0:1787826211.031863:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:3.0:1787826211.031865:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:3.0:1787826211.031867:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:3.0:1787826211.031869:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:3.0:1787826211.031870:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:3.0:1787826211.031873:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:3.0:1787826211.031874:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:3.0:1787826211.031877:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:3.0:1787826211.031878:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:3.0:1787826211.031880:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:3.0:1787826211.031883:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:3.0:1787826211.031885:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:3.0:1787826211.031887:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:3.0:1787826211.031890:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:3.0:1787826211.031892:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:3.0:1787826211.031893:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:3.0:1787826211.031895:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:3.0:1787826211.031897:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:3.0:1787826211.031899:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:3.0:1787826211.031900:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:3.0:1787826211.031911:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:3.0:1787826211.031912:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:3.0:1787826211.031914:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:3.0:1787826211.031916:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:3.0:1787826211.031918:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:3.0:1787826211.031920:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:3.0:1787826211.031922:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:3.0:1787826211.031924:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:3.0:1787826211.031926:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:3.0:1787826211.031928:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:3.0:1787826211.031930:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:3.0:1787826211.031932:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:3.0:1787826211.031934:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:3.0:1787826211.031936:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:3.0:1787826211.031938:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:3.0:1787826211.031939:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:3.0:1787826211.031941:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:3.0:1787826211.031943:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:3.0:1787826211.031944:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:3.0:1787826211.031946:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:3.0:1787826211.031947:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:3.0:1787826211.031949:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:3.0:1787826211.031952:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:3.0:1787826211.031954:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:3.0:1787826211.031956:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:3.0:1787826211.031958:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:3.0:1787826211.031959:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:3.0:1787826211.031961:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:3.0:1787826211.031963:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:3.0:1787826211.031964:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:3.0:1787826211.031967:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:3.0:1787826211.031968:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:3.0:1787826211.031971:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:3.0:1787826211.031973:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:3.0:1787826211.031975:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:3.0:1787826211.031977:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:3.0:1787826211.031978:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:3.0:1787826211.031980:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:3.0:1787826211.031982:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:3.0:1787826211.031984:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:3.0:1787826211.031986:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:3.0:1787826211.031989:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:3.0:1787826211.031991:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:3.0:1787826211.031992:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:3.0:1787826211.031995:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:3.0:1787826211.031996:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:3.0:1787826211.031998:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:3.0:1787826211.032000:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:3.0:1787826211.032002:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:3.0:1787826211.032004:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:3.0:1787826211.032006:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:3.0:1787826211.032008:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:3.0:1787826211.032010:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:3.0:1787826211.032012:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:3.0:1787826211.032014:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:3.0:1787826211.032016:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:3.0:1787826211.032018:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:3.0:1787826211.032019:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:3.0:1787826211.032021:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:3.0:1787826211.032023:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:3.0:1787826211.032025:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:3.0:1787826211.032027:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:3.0:1787826211.032029:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:3.0:1787826211.032031:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:3.0:1787826211.032033:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:3.0:1787826211.032035:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:3.0:1787826211.032037:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:3.0:1787826211.032040:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:3.0:1787826211.032042:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:3.0:1787826211.032044:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:3.0:1787826211.032045:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:3.0:1787826211.032047:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:3.0:1787826211.032050:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:3.0:1787826211.032051:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:3.0:1787826211.032055:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:3.0:1787826211.032057:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:3.0:1787826211.032058:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:3.0:1787826211.032061:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:3.0:1787826211.032063:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:3.0:1787826211.032065:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:3.0:1787826211.032067:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:3.0:1787826211.032069:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:3.0:1787826211.032071:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:3.0:1787826211.032074:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:3.0:1787826211.032076:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:3.0:1787826211.032078:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:3.0:1787826211.032081:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:3.0:1787826211.032083:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:3.0:1787826211.032084:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:3.0:1787826211.032086:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:3.0:1787826211.032088:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:3.0:1787826211.032089:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:3.0:1787826211.032092:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:3.0:1787826211.032094:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:3.0:1787826211.032096:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:3.0:1787826211.032098:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:3.0:1787826211.032100:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:3.0:1787826211.032102:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:3.0:1787826211.032104:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:3.0:1787826211.032106:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:3.0:1787826211.032108:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:3.0:1787826211.032110:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:3.0:1787826211.032111:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:3.0:1787826211.032113:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:3.0:1787826211.032115:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:3.0:1787826211.032117:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:3.0:1787826211.032119:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:3.0:1787826211.032121:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:3.0:1787826211.032123:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:3.0:1787826211.032124:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:3.0:1787826211.032127:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:3.0:1787826211.032129:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:3.0:1787826211.032131:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:3.0:1787826211.032133:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:3.0:1787826211.032134:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:3.0:1787826211.032136:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:3.0:1787826211.032139:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:3.0:1787826211.032141:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:3.0:1787826211.032143:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:3.0:1787826211.032145:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:3.0:1787826211.032147:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:3.0:1787826211.032149:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:3.0:1787826211.032150:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:3.0:1787826211.033065:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:3.0:1787826211.033101:0:2350:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907da56f8480 00000100:00100000:3.0:1787826211.033149:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011342592:192.168.203.104@tcp:3:lfs.0 00000080:00200080:3.0:1787826211.215267:0:62924:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88) cmd=400866b1 arg=3 00000080:00200008:3.0:1787826211.215296:0:62924:0:(file.c:6313:ll_getattr_dentry()) START file f200b.sanity-flr:[0x200000404:0x1f2:0x0](ffff907d93315b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787826211.215380:0:62924:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88),name=f200b.sanity-flr 00000002:00010000:3.0:1787826211.215421:0:62924:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f2:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826211.215447:0:62924:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0eb56 expref: -99 pid: 62918 timeout: 0 lvb_type: 3 lru_score: 9 lru_type: 1 00010000:00010000:3.0:1787826211.215467:0:62924:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0eb56 expref: -99 pid: 62918 timeout: 0 lvb_type: 3 lru_score: 9 lru_type: 1 00000080:00200000:3.0:1787826211.215475:0:62924:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826211.215487:0:62924:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f2:0x0](ffff907d93315b88) 00000080:00010000:3.0:1787826211.215491:0:62924:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f2:0x0](ffff907d93315b88) for lock 0xe5f736940357b2bc 00000080:00010000:3.0:1787826211.215500:0:62924:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b2bc from it ffff9dd14308fcd8 00010000:00010000:3.0:1787826211.215504:0:62924:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0eb56 expref: -99 pid: 62918 timeout: 0 lvb_type: 3 lru_score: 9 lru_type: 1 00010000:00010000:3.0:1787826211.215514:0:62924:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0eb56 expref: -99 pid: 62918 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00000080:00200008:3.0:1787826211.215555:0:62924:0:(file.c:2485:do_file_read_iter()) START file f200b.sanity-flr:[0x200000404:0x1f2:0x0], ppos: 9163072, count: 67108864 00000080:00200000:3.0:1787826211.215575:0:62924:0:(file.c:2090:ll_file_io_generic()) f200b.sanity-flr: read ppos: 9163072, bytes: 67108864 00000080:00200000:3.0:1787826211.215587:0:62924:0:(file.c:1848:ll_io_set_mirror()) f200b.sanity-flr: desiginated mirror: 3 00000020:00200000:3.0:1787826211.215597:0:62924:0:(cl_io.c:218:cl_io_rw_init()) header@00000000d9121b18[0x110, 2, [0x200000404:0x1f2:0x0] hash] 00000020:00200000:3.0:1787826211.215600:0:62924:0:(cl_io.c:218:cl_io_rw_init()) io range: 1 [9163072, 76271936) 0 0 00000080:00200000:3.0:1787826211.215627:0:62924:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f2:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787826211.215649:0:62924:0:(file.c:2170:ll_file_io_generic()) Range lock [2237, 18621] 00020000:00200000:3.0:1787826211.215659:0:62924:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 2 chunk: [9163072, 12582912) 76271936, 3419840 00020000:00200000:3.0:1787826211.215663:0:62924:0:(lov_io.c:1253:lov_io_iter_init()) component[12] flags 0x10 00000080:00200000:3.0:1787826211.215686:0:62924:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [2237, 3071] 00000008:00010000:3.0:1787826211.215781:0:62924:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:3.0:1787826211.215787:0:62924:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d98a1c230, osc lock ffff907d963b8af8, flags 0x0 00000080:00200000:3.0:1787826211.215797:0:62924:0:(vvp_io.c:874:vvp_io_read_start()) f200b.sanity-flr: read [9163072, 12582912) 00000001:00010000:3.0:1787826211.215813:0:62924:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1f2:0x0] 00000008:00010000:3.0:1787826211.215825:0:62924:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826211.215828:0:62924:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d92bef830, osc lock ffff907d963b8750, flags 0x400001002 00000008:00010000:3.0:1787826211.215834:0:62924:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826211.215835:0:62924:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d92bef870, osc lock ffff907d963b9248, flags 0x400001002 00000008:00010000:3.0:1787826211.215838:0:62924:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826211.215840:0:62924:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d92bef8b0, osc lock ffff907d963b9380, flags 0x400001002 00000008:00010000:3.0:1787826211.215847:0:62924:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826211.215848:0:62924:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d92bef8f0, osc lock ffff907d963b9998, flags 0x400001002 00000008:00010000:3.0:1787826211.215855:0:62924:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826211.215856:0:62924:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d92bef930, osc lock ffff907d963b8c30, flags 0x400001002 00000008:00010000:3.0:1787826211.215942:0:62924:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826211.215945:0:62924:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d92bef970, osc lock ffff907d963b8ea0, flags 0x400001002 00010000:00010000:3.0:1787826211.216010:0:62924:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000f62b7176/0xe5f736940357b31e lrc: 3/1,0 mode: --/PR res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.216022:0:62924:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000f62b7176/0xe5f736940357b31e lrc: 3/1,0 mode: --/PR res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->4194303] (req 0->4194303) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826211.216086:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1787826211.216114:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011343360:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826211.216750:0:62924:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c5e5d664/0xe5f736940357b325 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.216762:0:62924:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c5e5d664/0xe5f736940357b325 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [4194304->8388607] (req 4194304->8388607) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826211.216798:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011343488:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826211.216860:0:62924:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000e19e8cee/0xe5f736940357b32c lrc: 3/1,0 mode: --/PR res: [0x280000400:0x2ff:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.217221:0:62924:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000e19e8cee/0xe5f736940357b32c lrc: 3/1,0 mode: --/PR res: [0x280000400:0x2ff:0x0].0x0 rrc: 2 type: EXT [4194304->33554431] (req 4194304->33554431) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.217265:0:62924:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c633a10d/0xe5f736940357b25a lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x319:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0eaae expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:3.0:1787826211.217283:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1787826211.217292:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011343616:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826211.217592:0:62924:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (4194304 33554431) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c633a10d/0xe5f736940357b25a lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x319:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0eaae expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:3.0:1787826211.217745:0:62924:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00010000:00010000:3.0:1787826211.217751:0:62924:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c633a10d/0xe5f736940357b25a lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x319:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0eaae expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826211.217763:0:62924:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c633a10d/0xe5f736940357b25a lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x319:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0eaae expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1787826211.217790:0:62924:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000e79f459f/0xe5f736940357b261 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0eab5 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826211.217800:0:62924:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (33554432 18446744073709551615) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000e79f459f/0xe5f736940357b261 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 33554432->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0eab5 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:3.0:1787826211.217809:0:62924:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00010000:00010000:3.0:1787826211.217813:0:62924:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000e79f459f/0xe5f736940357b261 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 33554432->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0eab5 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826211.217822:0:62924:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000e79f459f/0xe5f736940357b261 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 33554432->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0eab5 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1787826211.217838:0:62924:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000ca4205d0/0xe5f736940357b268 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x31c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0eabc expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826211.217848:0:62924:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (33554432 18446744073709551615) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000ca4205d0/0xe5f736940357b268 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x31c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 33554432->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0eabc expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:3.0:1787826211.217856:0:62924:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00010000:00010000:3.0:1787826211.217860:0:62924:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000ca4205d0/0xe5f736940357b268 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x31c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 33554432->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0eabc expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826211.218005:0:62924:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000ca4205d0/0xe5f736940357b268 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x31c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 33554432->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0eabc expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:3.0:1787826211.218015:0:62924:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00010000:00010000:3.0:1787826211.243379:0:2350:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000e19e8cee/0xe5f736940357b32c lrc: 6/2,0 mode: --/PR res: [0x280000400:0x2ff:0x0].0x0 rrc: 2 type: EXT [4194304->33554431] (req 4194304->33554431) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.243407:0:2350:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000e19e8cee/0xe5f736940357b32c lrc: 6/2,0 mode: --/PR res: [0x280000400:0x2ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec67 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.243421:0:2350:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000e19e8cee/0xe5f736940357b32c lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x2ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ec67 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.243434:0:2350:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000e19e8cee/0xe5f736940357b32c lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x2ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec67 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:3.0:1787826211.243450:0:2350:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 4968768, setting rss=4968768; kms=4968768, end=18446744073709551615 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000e19e8cee/0xe5f736940357b32c lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x2ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0ec67 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:3.0:1787826211.243461:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00010000:00010000:3.0:1787826211.243466:0:2350:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000e19e8cee/0xe5f736940357b32c lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x2ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ec67 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.243485:0:2350:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000e19e8cee/0xe5f736940357b32c lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x2ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ec67 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.243493:0:2350:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000e19e8cee/0xe5f736940357b32c lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x2ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ec67 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.243502:0:2350:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000e19e8cee/0xe5f736940357b32c lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x2ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ec67 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1787826211.243521:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83527c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011343616:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826211.243545:0:2350:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c5e5d664/0xe5f736940357b325 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [4194304->8388607] (req 4194304->8388607) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.243555:0:2350:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c5e5d664/0xe5f736940357b325 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec60 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.243563:0:2350:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c5e5d664/0xe5f736940357b325 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ec60 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.243571:0:2350:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c5e5d664/0xe5f736940357b325 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec60 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:3.0:1787826211.243581:0:2350:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 8388608, setting rss=8388608; kms=8388608, end=18446744073709551615 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c5e5d664/0xe5f736940357b325 lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0ec60 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:3.0:1787826211.243590:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826211.243593:0:2350:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c5e5d664/0xe5f736940357b325 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ec60 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.243602:0:2350:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c5e5d664/0xe5f736940357b325 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ec60 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.243644:0:2350:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c5e5d664/0xe5f736940357b325 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ec60 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.243652:0:2350:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c5e5d664/0xe5f736940357b325 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x2fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ec60 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1787826211.243664:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011343488:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826211.243678:0:2350:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000f62b7176/0xe5f736940357b31e lrc: 6/2,0 mode: --/PR res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->4194303] (req 0->4194303) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.243687:0:2350:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000f62b7176/0xe5f736940357b31e lrc: 6/2,0 mode: --/PR res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec59 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.243706:0:2350:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000f62b7176/0xe5f736940357b31e lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ec59 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.243715:0:2350:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000f62b7176/0xe5f736940357b31e lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec59 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:3.0:1787826211.243724:0:2350:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 4194304, setting rss=4194304; kms=4194304, end=18446744073709551615 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000f62b7176/0xe5f736940357b31e lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0ec59 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:3.0:1787826211.243739:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:3.0:1787826211.243753:0:2350:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000f62b7176/0xe5f736940357b31e lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ec59 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.243762:0:2350:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000f62b7176/0xe5f736940357b31e lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ec59 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.243769:0:2350:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000f62b7176/0xe5f736940357b31e lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ec59 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826211.243777:0:2350:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000f62b7176/0xe5f736940357b31e lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x2f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ec59 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1787826211.243786:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011343360:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826213.630539:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011343744:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826213.630634:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011344000:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826213.630665:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1787826213.630680:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011343872:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826213.630702:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011344128:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826213.639290:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83527c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011343872:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826213.639309:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011344128:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826213.639317:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011343744:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826213.639328:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011344000:192.168.203.104@tcp:400:kworker.0 00000080:00200000:3.0:1787826218.265451:0:63343:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826218.265486:0:63343:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826218.265507:0:63343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d90874a00/0xe5f7369403578b71 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0a4d8 expref: -99 pid: 62013 timeout: 0 lvb_type: 0 lru_score: 122 lru_type: 1 00010000:00010000:3.0:1787826218.265524:0:63343:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d90874a00/0xe5f7369403578b71 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0a4d8 expref: -99 pid: 62013 timeout: 0 lvb_type: 0 lru_score: 122 lru_type: 1 00000080:00200000:3.0:1787826218.265532:0:63343:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826218.265541:0:63343:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826218.265545:0:63343:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f7369403578b71 00000080:00010000:3.0:1787826218.265552:0:63343:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f7369403578b71 from it ffff9dd140d53b58 00010000:00010000:3.0:1787826218.265556:0:63343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d90874a00/0xe5f7369403578b71 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0a4d8 expref: -99 pid: 62013 timeout: 0 lvb_type: 0 lru_score: 122 lru_type: 1 00010000:00010000:3.0:1787826218.265565:0:63343:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d90874a00/0xe5f7369403578b71 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0a4d8 expref: -99 pid: 62013 timeout: 0 lvb_type: 0 lru_score: 123 lru_type: 1 00000080:00200000:3.0:1787826218.265581:0:63343:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826218.265588:0:63343:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f200b.sanity-flr, flags=8 00000080:00200008:3.0:1787826218.265596:0:63343:0:(file.c:6313:ll_getattr_dentry()) START file f200b.sanity-flr:[0x200000404:0x1f2:0x0](ffff907d93315b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787826218.265600:0:63343:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88),name=f200b.sanity-flr 00000002:00010000:3.0:1787826218.265998:0:63343:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f2:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826218.266010:0:63343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0eb56 expref: -99 pid: 62918 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00010000:00010000:3.0:1787826218.266020:0:63343:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0eb56 expref: -99 pid: 62918 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00000080:00200000:3.0:1787826218.266027:0:63343:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826218.266032:0:63343:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f2:0x0](ffff907d93315b88) 00000080:00010000:3.0:1787826218.266034:0:63343:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f2:0x0](ffff907d93315b88) for lock 0xe5f736940357b2bc 00000080:00010000:3.0:1787826218.266038:0:63343:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b2bc from it ffff9dd140d53cb8 00010000:00010000:3.0:1787826218.266041:0:63343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0eb56 expref: -99 pid: 62918 timeout: 0 lvb_type: 3 lru_score: 10 lru_type: 1 00010000:00010000:3.0:1787826218.266047:0:63343:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0eb56 expref: -99 pid: 62918 timeout: 0 lvb_type: 3 lru_score: 11 lru_type: 1 00000080:00200000:3.0:1787826218.266075:0:63343:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826218.266083:0:63343:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826218.266090:0:63343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d90874a00/0xe5f7369403578b71 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0a4d8 expref: -99 pid: 62013 timeout: 0 lvb_type: 0 lru_score: 123 lru_type: 1 00010000:00010000:3.0:1787826218.266098:0:63343:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d90874a00/0xe5f7369403578b71 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0a4d8 expref: -99 pid: 62013 timeout: 0 lvb_type: 0 lru_score: 123 lru_type: 1 00000080:00200000:3.0:1787826218.266104:0:63343:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826218.266108:0:63343:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826218.266111:0:63343:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f7369403578b71 00000080:00010000:3.0:1787826218.266115:0:63343:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f7369403578b71 from it ffff9dd140d53c10 00010000:00010000:3.0:1787826218.266118:0:63343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d90874a00/0xe5f7369403578b71 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0a4d8 expref: -99 pid: 62013 timeout: 0 lvb_type: 0 lru_score: 123 lru_type: 1 00010000:00010000:3.0:1787826218.266127:0:63343:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d90874a00/0xe5f7369403578b71 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0a4d8 expref: -99 pid: 62013 timeout: 0 lvb_type: 0 lru_score: 124 lru_type: 1 00000080:00200000:3.0:1787826218.266136:0:63343:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826218.266143:0:63343:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f200b.sanity-flr, flags=0 00000080:00200000:3.0:1787826218.266146:0:63343:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826218.266152:0:63343:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826218.266158:0:63343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d90874a00/0xe5f7369403578b71 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0a4d8 expref: -99 pid: 62013 timeout: 0 lvb_type: 0 lru_score: 124 lru_type: 1 00010000:00010000:3.0:1787826218.266166:0:63343:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d90874a00/0xe5f7369403578b71 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0a4d8 expref: -99 pid: 62013 timeout: 0 lvb_type: 0 lru_score: 124 lru_type: 1 00000080:00200000:3.0:1787826218.266172:0:63343:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826218.266176:0:63343:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826218.266179:0:63343:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f7369403578b71 00000080:00010000:3.0:1787826218.266183:0:63343:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f7369403578b71 from it ffff9dd140d53d68 00010000:00010000:3.0:1787826218.266185:0:63343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d90874a00/0xe5f7369403578b71 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0a4d8 expref: -99 pid: 62013 timeout: 0 lvb_type: 0 lru_score: 124 lru_type: 1 00010000:00010000:3.0:1787826218.266193:0:63343:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d90874a00/0xe5f7369403578b71 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0a4d8 expref: -99 pid: 62013 timeout: 0 lvb_type: 0 lru_score: 125 lru_type: 1 00000080:00200000:3.0:1787826218.266202:0:63343:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 3 00000080:00200000:3.0:1787826218.266207:0:63343:0:(namei.c:2418:ll_unlink()) VFS Op:name=f200b.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148) 00010000:00010000:3.0:1787826218.266235:0:63343:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d90874a00/0xe5f7369403578b71 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc0a4d8 expref: -99 pid: 62013 timeout: 0 lvb_type: 0 lru_score: 125 lru_type: 1 00010000:00010000:3.0:1787826218.266278:0:63343:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19e00/0xe5f736940357b2ed lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f2:0x0].0x0 bits 0xd/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0ebd4 expref: -99 pid: 62921 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000002:00100000:3.0:1787826218.266338:0:63343:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d9114b480 x1874670011339904/t8589942858(8589942858) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/2312 e 0 to 0 dl 1787826226 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1787826218.266349:0:63343:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1787826218.266355:0:63343:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83524f00 x1874670011344384/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1787826218.266377:0:63343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:63343:1874670011344384:192.168.203.104@tcp:35:rm.0 00000100:00100000:3.0:1787826218.266483:0:63343:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d871eecc0 going to sleep for 11 seconds 00000100:00100000:3.0:1787826218.275123:0:63343:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589942858 gen 1 00000100:00100000:3.0:1787826218.275147:0:63343:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff907d83524f00 x1874670011344384/t8589942859(8589942859) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/456 e 0 to 0 dl 1787826234 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000100:00100000:3.0:1787826218.275162:0:63343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:63343:1874670011344384:192.168.203.104@tcp:35:rm.0 00000002:00100000:3.0:1787826218.275193:0:63343:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d9114b480 x1874670011339904/t8589942858(8589942858) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/2312 e 0 to 0 dl 1787826226 ref 1 fl Complete:RMQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826218.275256:0:63343:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x1f2:0x0]: losing layout lock 00010000:00010000:3.0:1787826218.275282:0:63343:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b2ae lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f2:0x0].0x0 bits 0x8/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0eb10 expref: -99 pid: 62909 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1787826218.275295:0:63343:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x1f2:0x0]: losing layout lock 00010000:00010000:3.0:1787826218.275303:0:63343:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0eb56 expref: -99 pid: 62918 timeout: 0 lvb_type: 3 lru_score: 11 lru_type: 1 00000080:00200000:3.0:1787826218.275317:0:63343:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x1f2:0x0]: losing layout lock 00010000:00010000:3.0:1787826218.275336:0:63343:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb000/0xe5f7369403578bd3 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x20/0x0 rrc: 6 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc0a683 expref: -99 pid: 62027 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826218.275388:0:63343:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19e00/0xe5f736940357b2ed lrc: 1/0,0 mode: --/CR res: [0x200000404:0x1f2:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0ebd4 expref: -99 pid: 62921 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826218.275397:0:63343:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b2ae lrc: 1/0,0 mode: --/CR res: [0x200000404:0x1f2:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0eb10 expref: -99 pid: 62909 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826218.275405:0:63343:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 1/0,0 mode: --/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0eb56 expref: -99 pid: 62918 timeout: 0 lvb_type: 3 lru_score: 11 lru_type: 1 00010000:00010000:3.0:1787826218.275412:0:63343:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb000/0xe5f7369403578bd3 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x20/0x0 rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0a683 expref: -99 pid: 62027 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826218.275420:0:63343:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d90874a00/0xe5f7369403578b71 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0a4d8 expref: -99 pid: 62013 timeout: 0 lvb_type: 0 lru_score: 125 lru_type: 1 00010000:00010000:3.0:1787826218.275426:0:63343:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 5 locks packed 00010000:00010000:3.0:1787826218.275439:0:63343:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19e00/0xe5f736940357b2ed lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1f2:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0ebd4 expref: -99 pid: 62921 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826218.275793:0:63343:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b2ae lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1f2:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0eb10 expref: -99 pid: 62909 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826218.275805:0:63343:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b2bc lrc: 0/0,0 mode: --/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0eb56 expref: -99 pid: 62918 timeout: 0 lvb_type: 3 lru_score: 11 lru_type: 1 00010000:00010000:3.0:1787826218.275819:0:63343:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb000/0xe5f7369403578bd3 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x1f2:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0a683 expref: -99 pid: 62027 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826218.275833:0:63343:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d90874a00/0xe5f7369403578b71 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0a4d8 expref: -99 pid: 62013 timeout: 0 lvb_type: 0 lru_score: 125 lru_type: 1 00000020:00100000:3.0:1787826218.275860:0:63343:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1787826218.275869:0:63343:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83524f00 x1874670011344640/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 528/2200 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826218.275893:0:63343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:63343:1874670011344640:192.168.203.104@tcp:36:rm.0 00000100:00100000:3.0:1787826218.276198:0:63343:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d871eecc0 going to sleep for 11 seconds 00000100:00100000:3.0:1787826218.288841:0:63343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:63343:1874670011344640:192.168.203.104@tcp:36:rm.0 00000080:00200000:3.0:1787826218.288940:0:63343:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f2:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1787826218.288957:0:63343:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1f2:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 3 00020000:00200000:3.0:1787826218.288961:0:63343:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:3.0:1787826218.288969:0:63343:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1787826218.288983:0:63343:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:3.0:1787826218.288997:0:63343:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x10 00020000:00200000:3.0:1787826218.289007:0:63343:0:(lov_io.c:1253:lov_io_iter_init()) component[3] flags 0x10 00020000:00200000:3.0:1787826218.289016:0:63343:0:(lov_io.c:1253:lov_io_iter_init()) component[4] flags 0x10 00000080:00200000:3.0:1787826218.289062:0:63343:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f2:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787826218.289072:0:63343:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200000404:0x1f2:0x0](ffff907d93315b88) 00000008:00010000:3.0:1787826218.289120:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 1048576 blocks 2000 [cma]time: 1787826208 1787826208 1787826173 by oinfo size 1048576 blocks 2000 [cma]time 1787826208 1787826208 1787826173 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b2c3 lrc: 1/0,0 mode: PR/PR res: [0x280000400:0x2f7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0eb9c expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:3.0:1787826218.289161:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 2097152 blocks 2048 [cma]time: 1787826208 1787826208 1787826173 by oinfo size 2097152 blocks 2048 [cma]time 1787826208 1787826208 1787826173 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000dd58a055/0xe5f736940357b2ca lrc: 1/0,0 mode: PR/PR res: [0x240000400:0x313:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ebaa expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:3.0:1787826218.289178:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 2097152 blocks 2056 [cma]time: 1787826208 1787826208 1787826188 by oinfo size 2097152 blocks 2056 [cma]time 1787826208 1787826208 1787826188 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b55f8210/0xe5f736940357b2d1 lrc: 1/0,0 mode: PR/PR res: [0x240000400:0x314:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0eba3 expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:3.0:1787826218.289194:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 2097152 blocks 2048 [cma]time: 1787826208 1787826208 1787826190 by oinfo size 2097152 blocks 2048 [cma]time 1787826208 1787826208 1787826190 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000aad1631b/0xe5f736940357b2d8 lrc: 1/0,0 mode: PR/PR res: [0x280000400:0x2f9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ebb1 expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:3.0:1787826218.289209:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 4968768 blocks 5624 [cma]time: 1787826208 1787826208 1787826194 by oinfo size 4968768 blocks 5624 [cma]time 1787826208 1787826208 1787826210 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000002d53f219/0xe5f736940357b2df lrc: 1/0,0 mode: PR/PR res: [0x280000400:0x2fa:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ebb8 expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:3.0:1787826218.289223:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 4194304 blocks 4104 [cma]time: 1787826208 1787826208 1787826193 by oinfo size 4194304 blocks 4104 [cma]time 1787826208 1787826208 1787826193 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c5159d29/0xe5f736940357b2e6 lrc: 1/0,0 mode: PR/PR res: [0x240000400:0x315:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ebbf expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:3.0:1787826218.289239:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 1787826208 1787826208 0 by oinfo size 0 blocks 0 [cma]time 1787826208 1787826208 0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000a49bef08/0xe5f736940357b230 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x300:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ea76 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000008:00010000:3.0:1787826218.289253:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 1787826208 1787826208 0 by oinfo size 0 blocks 0 [cma]time 1787826208 1787826208 0 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000888724c1/0xe5f736940357b237 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x31a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ea7d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000008:00010000:3.0:1787826218.289269:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 2097152 blocks 4040 [cma]time: 1787826208 1787826203 1787826202 by oinfo size 2097152 blocks 4040 [cma]time 1787826208 1787826203 1787826202 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b2f4 lrc: 1/0,0 mode: PR/PR res: [0x240000400:0x312:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ec1a expref: -99 pid: 62921 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:3.0:1787826218.289286:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 3145728 blocks 4096 [cma]time: 1787826208 1787826203 1787826202 by oinfo size 3145728 blocks 4096 [cma]time 1787826208 1787826203 1787826202 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1dd66eb/0xe5f736940357b2fb lrc: 1/0,0 mode: PR/PR res: [0x240000400:0x316:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ec21 expref: -99 pid: 62921 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:3.0:1787826218.289302:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 3145728 blocks 4096 [cma]time: 1787826208 1787826203 1787826202 by oinfo size 3145728 blocks 4096 [cma]time 1787826208 1787826203 1787826202 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001b7e234c/0xe5f736940357b302 lrc: 1/0,0 mode: PR/PR res: [0x280000400:0x2fb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ec36 expref: -99 pid: 62921 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:3.0:1787826218.289316:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 4194304 blocks 2048 [cma]time: 1787826208 1787826203 1787826202 by oinfo size 4194304 blocks 2048 [cma]time 1787826208 1787826203 1787826202 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b309 lrc: 1/0,0 mode: PR/PR res: [0x240000400:0x317:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ec28 expref: -99 pid: 62921 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:3.0:1787826218.289330:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 4194304 blocks 2048 [cma]time: 1787826208 1787826203 1787826202 by oinfo size 4194304 blocks 2048 [cma]time 1787826208 1787826203 1787826202 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b310 lrc: 1/0,0 mode: PR/PR res: [0x280000400:0x2fc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ec3d expref: -99 pid: 62921 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:3.0:1787826218.289345:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 4968768 blocks 1520 [cma]time: 1787826208 1787826203 1787826202 by oinfo size 4968768 blocks 1520 [cma]time 1787826208 1787826203 1787826210 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b317 lrc: 1/0,0 mode: PR/PR res: [0x240000400:0x318:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ec2f expref: -99 pid: 62921 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:3.0:1787826218.289361:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 4194304 blocks 0 [cma]time: 1787826208 1787826203 1787826202 by oinfo size 4194304 blocks 0 [cma]time 1787826208 1787826203 1787826202 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000bea3d6dd/0xe5f736940357b299 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x2fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0eaf4 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000008:00010000:3.0:1787826218.289377:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 1787826208 1787826203 1787826202 by oinfo size 0 blocks 0 [cma]time 1787826208 1787826203 1787826202 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b2a0 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x301:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0eafb expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000008:00010000:3.0:1787826218.289393:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 1787826208 1787826203 1787826202 by oinfo size 0 blocks 0 [cma]time 1787826208 1787826203 1787826202 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e731006c/0xe5f736940357b2a7 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x31b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0eb02 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000008:00010000:3.0:1787826218.289409:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 4194304 blocks 8136 [cma]time: 1787826208 1787826208 0 by oinfo size 4194304 blocks 8136 [cma]time 1787826208 1787826208 0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000f62b7176/0xe5f736940357b31e lrc: 1/0,0 mode: PR/PR res: [0x280000400:0x2f8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ec59 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:3.0:1787826218.289423:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 8388608 blocks 8192 [cma]time: 1787826208 1787826208 0 by oinfo size 8388608 blocks 8192 [cma]time 1787826208 1787826208 0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c5e5d664/0xe5f736940357b325 lrc: 1/0,0 mode: PR/PR res: [0x280000400:0x2fe:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ec60 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:3.0:1787826218.290638:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 4968768 blocks 1520 [cma]time: 1787826208 1787826208 0 by oinfo size 4968768 blocks 1520 [cma]time 1787826208 1787826208 1787826211 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000e19e8cee/0xe5f736940357b32c lrc: 1/0,0 mode: PR/PR res: [0x280000400:0x2ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ec67 expref: -99 pid: 62924 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:3.0:1787826218.290659:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 4194304 blocks 0 [cma]time: 1787826208 1787826208 0 by oinfo size 4194304 blocks 0 [cma]time 1787826208 1787826208 0 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c633a10d/0xe5f736940357b25a lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x319:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0eaae expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000008:00010000:3.0:1787826218.290677:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 1787826208 1787826208 0 by oinfo size 0 blocks 0 [cma]time 1787826208 1787826208 0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000e79f459f/0xe5f736940357b261 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0eab5 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000008:00010000:3.0:1787826218.290695:0:63343:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 1787826208 1787826208 0 by oinfo size 0 blocks 0 [cma]time 1787826208 1787826208 0 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000ca4205d0/0xe5f736940357b268 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x31c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 33554432->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0eabc expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1787826218.290831:0:63343:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826218.290873:0:63343:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826218.290973:0:63343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.290982:0:63343:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.290994:0:63343:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826218.291031:0:63343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:63343:1874670011344768:192.168.203.104@tcp:101:rm.0 00000100:00100000:3.0:1787826218.291171:0:63343:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d871eecc0 going to sleep for 11 seconds 00000100:00100000:3.0:1787826218.306394:0:63343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:63343:1874670011344768:192.168.203.104@tcp:101:rm.0 00010000:00010000:3.0:1787826218.306443:0:63343:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.306459:0:63343:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.306473:0:63343:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.306490:0:63343:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.306503:0:63343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.306512:0:63343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.306519:0:63343:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826218.306532:0:63343:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d835270c0 x1874670011344768/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/688 e 0 to 0 dl 1787826234 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1787826218.306548:0:63343:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.306563:0:63343:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1787826218.306571:0:63343:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1787826218.306661:0:63343:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:3.0:1787826218.306669:0:63343:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b333 00000080:00010000:3.0:1787826218.306675:0:63343:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826218.306678:0:63343:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b333 from it ffff9dd140d53b58 00010000:00010000:3.0:1787826218.306681:0:63343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.306694:0:63343:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787826218.306725:0:63343:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826218.306776:0:63343:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f200b.sanity-flr-2, dir=[0x200000007:0x1:0x0](ffff907d84619148), flags=8 00000080:00200000:3.0:1787826218.306781:0:63343:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f200b.sanity-flr-2, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=getattr 00000002:00010000:3.0:1787826218.306806:0:63343:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f200b.sanity-flr-2,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:3.0:1787826218.306865:0:63343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b33a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.306873:0:63343:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b33a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.306883:0:63343:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b33a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826218.306909:0:63343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:63343:1874670011344896:192.168.203.104@tcp:101:rm.0 00000100:00100000:3.0:1787826218.307024:0:63343:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d871eecc0 going to sleep for 11 seconds 00000100:00100000:3.0:1787826218.311695:0:63343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:63343:1874670011344896:192.168.203.104@tcp:101:rm.0 00010000:00010000:3.0:1787826218.311743:0:63343:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b33a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.311762:0:63343:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b33a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.311769:0:63343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b33a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.311783:0:63343:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b33a lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.311795:0:63343:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b33a lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.311803:0:63343:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d86043e00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826218.311809:0:63343:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b33a lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.311818:0:63343:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b33a lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.311827:0:63343:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b33a lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.311836:0:63343:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826218.311846:0:63343:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b33a lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.311863:0:63343:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b33a lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826218.311888:0:63343:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff907d835270c0 x1874670011344896/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/584 e 0 to 0 dl 1787826234 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826218.311908:0:63343:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:3.0:1787826218.311948:0:63343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826218.311964:0:63343:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00010000:3.0:1787826218.311976:0:63343:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b333 from it ffff9dd140d53930 00010000:00010000:3.0:1787826218.311981:0:63343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826218.311990:0:63343:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787826218.312079:0:63343:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826218.312098:0:63343:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826218.312107:0:63343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826218.312116:0:63343:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787826218.312123:0:63343:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826218.312129:0:63343:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826218.312133:0:63343:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b333 00000080:00010000:3.0:1787826218.312138:0:63343:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b333 from it ffff9dd140d53c10 00010000:00010000:3.0:1787826218.312142:0:63343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826218.312151:0:63343:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1787826218.312161:0:63343:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826218.312174:0:63343:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f200b.sanity-flr-2, flags=0 00000080:00200000:3.0:1787826218.312200:0:63343:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826218.312207:0:63343:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826218.312213:0:63343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826218.312222:0:63343:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1787826218.312229:0:63343:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826218.312233:0:63343:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826218.312236:0:63343:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b333 00000080:00010000:3.0:1787826218.312240:0:63343:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b333 from it ffff9dd140d53b58 00010000:00010000:3.0:1787826218.312244:0:63343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826218.312252:0:63343:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1787826218.312261:0:63343:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826218.312280:0:63343:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f200b.sanity-flr-3, dir=[0x200000007:0x1:0x0](ffff907d84619148), flags=8 00000080:00200000:3.0:1787826218.312284:0:63343:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f200b.sanity-flr-3, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=getattr 00000002:00010000:3.0:1787826218.312298:0:63343:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f200b.sanity-flr-3,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:3.0:1787826218.312349:0:63343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043a00/0xe5f736940357b341 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.312357:0:63343:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043a00/0xe5f736940357b341 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.312370:0:63343:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043a00/0xe5f736940357b341 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826218.312401:0:63343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:63343:1874670011345024:192.168.203.104@tcp:101:rm.0 00000100:00100000:3.0:1787826218.312521:0:63343:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d871eecc0 going to sleep for 11 seconds 00000100:00100000:3.0:1787826218.316028:0:63343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:63343:1874670011345024:192.168.203.104@tcp:101:rm.0 00010000:00010000:3.0:1787826218.316061:0:63343:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043a00/0xe5f736940357b341 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.316078:0:63343:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043a00/0xe5f736940357b341 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.316086:0:63343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043a00/0xe5f736940357b341 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.316093:0:63343:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043a00/0xe5f736940357b341 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.316103:0:63343:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043a00/0xe5f736940357b341 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.316110:0:63343:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d86043a00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826218.316115:0:63343:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043a00/0xe5f736940357b341 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.316122:0:63343:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043a00/0xe5f736940357b341 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.316130:0:63343:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043a00/0xe5f736940357b341 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.316138:0:63343:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826218.316147:0:63343:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043a00/0xe5f736940357b341 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826218.316154:0:63343:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043a00/0xe5f736940357b341 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 63343 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826218.316177:0:63343:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff907d835270c0 x1874670011345024/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/584 e 0 to 0 dl 1787826234 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826218.316194:0:63343:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:3.0:1787826218.316226:0:63343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1787826218.316242:0:63343:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00010000:3.0:1787826218.316252:0:63343:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b333 from it ffff9dd140d53930 00010000:00010000:3.0:1787826218.316256:0:63343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1787826218.316265:0:63343:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1787826218.316351:0:63343:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826218.316369:0:63343:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826218.316377:0:63343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1787826218.316385:0:63343:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1787826218.316398:0:63343:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826218.316403:0:63343:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826218.316407:0:63343:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b333 00000080:00010000:3.0:1787826218.316412:0:63343:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b333 from it ffff9dd140d53c10 00010000:00010000:3.0:1787826218.316415:0:63343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1787826218.316424:0:63343:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b333 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ec91 expref: -99 pid: 63343 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1787826218.316433:0:63343:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826218.316448:0:63343:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f200b.sanity-flr-3, flags=0 00000100:00100000:3.0:1787826218.741435:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011345152:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826218.741531:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011345408:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826218.741565:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1787826218.741571:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011345280:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826218.751217:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011345152:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826218.759232:0:2350:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0001-osc-ffff907d986d0000: committing for last_committed 17179873783 gen 1 00000100:00100000:3.0:1787826218.759284:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011345408:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826218.759306:0:2350:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff907d986d0000: committing for last_committed 21474840444 gen 3 00000100:00100000:3.0:1787826218.759330:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83527c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011345280:192.168.203.104@tcp:400:kworker.0 00000001:02000400:3.0:1787826220.198805:0:63483:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-flr test 200c: layout change racing with open: LOVEA changes ========================================================== 06:23:38 (1787826218) 00000100:00100000:3.0:1787826220.289596:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826220.298979:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011345536:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1787826220.734992:0:63508:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job mount.lustre:f411759c-2893-4f83-9e11-e7e6f379c36e:63508:1874670011345920:192.168.203.104@tcp:501:mount.lustre.0 00000100:00100000:3.0:1787826220.735046:0:63508:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job mount.lustre:f411759c-2893-4f83-9e11-e7e6f379c36e:63508:1874670011346048:192.168.203.104@tcp:503:mount.lustre.0 00000100:00100000:3.0:1787826220.735127:0:63508:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d999376c0 going to sleep for 11 seconds 00000100:00100000:3.0:1787826220.820183:0:63508:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148000 pname:cluuid:pid:xid:nid:opc:job mount.lustre:f411759c-2893-4f83-9e11-e7e6f379c36e:63508:1874670011346432:192.168.203.104@tcp:101:mount.lustre.0 00010000:00010000:3.0:1787826220.820211:0:63508:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.104@tcp lock: ffff907d98a18e00/0xe5f736940357b372 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x7918213f4fc0ecd7 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826220.820224:0:63508:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.104@tcp lock: ffff907d98a18e00/0xe5f736940357b372 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x7918213f4fc0ecd7 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826220.820861:0:63508:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job mount.lustre:f411759c-2893-4f83-9e11-e7e6f379c36e:63508:1874670011346560:192.168.203.104@tcp:256:mount.lustre.0 00000100:00100000:3.0:1787826220.820943:0:63508:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c02f00 going to sleep for 11 seconds 00000100:00100000:3.0:1787826220.916973:0:2347:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff907d850a3000: committing for last_committed 0 gen 1 00000100:00080000:3.0:1787826220.916990:0:2347:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-OST0000-osc-ffff907d850a3000: connect to target with instance 5 00000100:00080000:3.0:1787826220.917001:0:2347:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-OST0000-osc-ffff907d850a3000: Resetting ns_connect_flags to server flags: 0xa0427af2e3440038 00000100:00080000:3.0:1787826220.917015:0:2347:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0000_UUID 00000100:00080000:3.0:1787826220.917018:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d86059000 lustre-OST0000_UUID: changing import state from CONNECTING to FULL 00020000:01000000:3.0:1787826220.917026:0:2347:0:(lov_obd.c:398:lov_set_osc_active()) Marking OSC lustre-OST0000_UUID active 00000080:00000004:3.0:1787826220.917031:0:2347:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0467af2e3640038 -> 0xa0427af2e3440038 00000080:00080000:3.0:1787826220.917036:0:2347:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:3.0:1787826220.917052:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d203faaf-7906-4d13-b1ba-8437467d6639:2347:1874670011347968:192.168.203.104@tcp:8:mount.lustre.0 00000100:00100000:3.0:1787826220.917069:0:2347:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0001-osc-ffff907d850a3000: committing for last_committed 0 gen 1 00000100:00080000:3.0:1787826220.917074:0:2347:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-OST0001-osc-ffff907d850a3000: connect to target with instance 4 00000100:00080000:3.0:1787826220.917079:0:2347:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-OST0001-osc-ffff907d850a3000: Resetting ns_connect_flags to server flags: 0xa0427af2e3440038 00000100:00080000:3.0:1787826220.917084:0:2347:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0001_UUID 00000100:00080000:3.0:1787826220.917086:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d82ba7000 lustre-OST0001_UUID: changing import state from CONNECTING to FULL 00020000:01000000:3.0:1787826220.917089:0:2347:0:(lov_obd.c:398:lov_set_osc_active()) Marking OSC lustre-OST0001_UUID active 00000080:00000004:3.0:1787826220.917091:0:2347:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0427af2e3440038 -> 0xa0427af2e3440038 00000080:00080000:3.0:1787826220.917094:0:2347:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:3.0:1787826220.917099:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d203faaf-7906-4d13-b1ba-8437467d6639:2347:1874670011348096:192.168.203.104@tcp:8:mount.lustre.0 00010000:00010000:3.0:1787826220.939951:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826221.187773:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000080:00200000:2.0:1787826221.276053:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 240 of 0 to 4096 bytes 00000080:00200000:2.0:1787826221.276068:0:63537:0:(rw26.c:875:ll_write_end()) pos 983040, len 4096, copied 4096 00000080:00200000:2.0:1787826221.276072:0:63537:0:(rw26.c:933:ll_write_end()) page@000000005eba2e16[3 0000000029ab925b 2 1 0000000008f41f32 0 0 240] 00000080:00200000:2.0:1787826221.276074:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000009baba0fd 17ffffc0001001 3:0 ffff907d96149d98 240 lru 00000080:00200000:2.0:1787826221.276078:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@0000000020f3e8f5 240: 1< 0 - - > 2< 983040 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 485711872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1787826221.276080:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000005eba2e16 00000080:00200000:2.0:1787826221.276081:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 241. 00000080:00200000:2.0:1787826221.276082:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 241 of 0 to 4096 bytes 00000080:00200000:2.0:1787826221.276093:0:63537:0:(rw26.c:875:ll_write_end()) pos 987136, len 4096, copied 4096 00000080:00200000:2.0:1787826221.276096:0:63537:0:(rw26.c:933:ll_write_end()) page@000000002f2b9e54[3 0000000029ab925b 2 1 0000000008f41f32 0 0 241] 00000080:00200000:2.0:1787826221.276099:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000957b0a9a 17ffffc0001001 3:0 ffff907d961491d0 241 lru 00000080:00200000:2.0:1787826221.276103:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000009db2eac8 241: 1< 0 - - > 2< 987136 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 485711872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1787826221.276105:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000002f2b9e54 00000080:00200000:2.0:1787826221.276106:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 242. 00000080:00200000:2.0:1787826221.276107:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 242 of 0 to 4096 bytes 00000080:00200000:2.0:1787826221.276117:0:63537:0:(rw26.c:875:ll_write_end()) pos 991232, len 4096, copied 4096 00000080:00200000:2.0:1787826221.276120:0:63537:0:(rw26.c:933:ll_write_end()) page@0000000014fda10d[3 0000000029ab925b 2 1 0000000008f41f32 0 0 242] 00000080:00200000:2.0:1787826221.276123:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000440401f0 17ffffc0001001 3:0 ffff907d96149bc8 242 lru 00000080:00200000:2.0:1787826221.276127:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000002d8a5005 242: 1< 0 - - > 2< 991232 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 485711872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1787826221.276129:0:63537:0:(rw26.c:933:ll_write_end()) end page@0000000014fda10d 00000080:00200000:2.0:1787826221.276130:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 243. 00000080:00200000:2.0:1787826221.276131:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 243 of 0 to 4096 bytes 00000080:00200000:2.0:1787826221.276141:0:63537:0:(rw26.c:875:ll_write_end()) pos 995328, len 4096, copied 4096 00000080:00200000:2.0:1787826221.276145:0:63537:0:(rw26.c:933:ll_write_end()) page@0000000080ba6f2d[3 0000000029ab925b 2 1 0000000008f41f32 0 0 243] 00000080:00200000:2.0:1787826221.276147:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000c8b5639f 17ffffc0001001 3:0 ffff907d96149740 243 lru 00000080:00200000:2.0:1787826221.276153:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000009bc51d8b 243: 1< 0 - - > 2< 995328 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 485711872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1787826221.276155:0:63537:0:(rw26.c:933:ll_write_end()) end page@0000000080ba6f2d 00000080:00200000:2.0:1787826221.276155:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 244. 00000080:00200000:2.0:1787826221.276157:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 244 of 0 to 4096 bytes 00000080:00200000:2.0:1787826221.276169:0:63537:0:(rw26.c:875:ll_write_end()) pos 999424, len 4096, copied 4096 00000080:00200000:2.0:1787826221.276173:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000cbe703c4[3 0000000029ab925b 2 1 0000000008f41f32 0 0 244] 00000080:00200000:2.0:1787826221.276176:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000bbcaf50d 17ffffc0001001 3:0 ffff907d961493a0 244 lru 00000080:00200000:2.0:1787826221.276182:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000000be37ae8 244: 1< 0 - - > 2< 999424 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 485711872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1787826221.276184:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000cbe703c4 00000080:00200000:2.0:1787826221.276185:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 245. 00000080:00200000:2.0:1787826221.276187:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 245 of 0 to 4096 bytes 00000080:00200000:2.0:1787826221.276201:0:63537:0:(rw26.c:875:ll_write_end()) pos 1003520, len 4096, copied 4096 00000080:00200000:2.0:1787826221.276205:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000478f1641[3 0000000029ab925b 2 1 0000000008f41f32 0 0 245] 00000080:00200000:2.0:1787826221.276208:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000003e0ab254 17ffffc0001001 3:0 ffff907d96149828 245 lru 00000080:00200000:2.0:1787826221.276213:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000001e4d97b1 245: 1< 0 - - > 2< 1003520 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 485711872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1787826221.276215:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000478f1641 00000080:00200000:2.0:1787826221.276217:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 246. 00000080:00200000:2.0:1787826221.276219:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 246 of 0 to 4096 bytes 00000080:00200000:2.0:1787826221.276246:0:63537:0:(rw26.c:875:ll_write_end()) pos 1007616, len 4096, copied 4096 00000080:00200000:2.0:1787826221.276251:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000b4823609[3 0000000029ab925b 2 1 0000000008f41f32 0 0 246] 00000080:00200000:2.0:1787826221.276253:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @0000000044336785 17ffffc0001021 2:0 ffff907d96149658 246 lru 00000080:00200000:2.0:1787826221.276259:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000b11a8552 246: 1< 0 - - > 2< 1007616 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 485711872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1787826221.276260:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000b4823609 00000080:00200000:2.0:1787826221.276261:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 247. 00000080:00200000:2.0:1787826221.276263:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 247 of 0 to 4096 bytes 00000080:00200000:2.0:1787826221.276278:0:63537:0:(rw26.c:875:ll_write_end()) pos 1011712, len 4096, copied 4096 00000080:00200000:2.0:1787826221.276282:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000fa7c7818[3 0000000029ab925b 2 1 0000000008f41f32 0 0 247] 00000080:00200000:2.0:1787826221.276285:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @0000000068a169b7 17ffffc0001001 3:0 ffff907d96149e80 247 lru 00000080:00200000:2.0:1787826221.276291:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000dadd3c72 247: 1< 0 - - > 2< 1011712 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 485711872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1787826221.276293:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000fa7c7818 00000080:00200000:2.0:1787826221.276294:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 248. 00000080:00200000:2.0:1787826221.276295:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 248 of 0 to 4096 bytes 00000080:00200000:2.0:1787826221.276311:0:63537:0:(rw26.c:875:ll_write_end()) pos 1015808, len 4096, copied 4096 00000080:00200000:2.0:1787826221.276315:0:63537:0:(rw26.c:933:ll_write_end()) page@000000007ac725f5[3 0000000029ab925b 2 1 0000000008f41f32 0 0 248] 00000080:00200000:2.0:1787826221.276318:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000787ebd80 17ffffc0001001 3:0 ffff907d96149000 248 lru 00000080:00200000:2.0:1787826221.276322:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@0000000021c27237 248: 1< 0 - - > 2< 1015808 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 485711872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1787826221.276324:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000007ac725f5 00000080:00200000:2.0:1787826221.276325:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 249. 00000080:00200000:2.0:1787826221.276327:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 249 of 0 to 4096 bytes 00000080:00200000:2.0:1787826221.276340:0:63537:0:(rw26.c:875:ll_write_end()) pos 1019904, len 4096, copied 4096 00000080:00200000:2.0:1787826221.276344:0:63537:0:(rw26.c:933:ll_write_end()) page@0000000010ca76ff[3 0000000029ab925b 2 1 0000000008f41f32 0 0 249] 00000080:00200000:2.0:1787826221.276347:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000b439ae5f 17ffffc0001001 3:0 ffff907d961499f8 249 lru 00000080:00200000:2.0:1787826221.276351:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000cd689aed 249: 1< 0 - - > 2< 1019904 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 485711872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1787826221.276353:0:63537:0:(rw26.c:933:ll_write_end()) end page@0000000010ca76ff 00000080:00200000:2.0:1787826221.276354:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 250. 00000080:00200000:2.0:1787826221.276356:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 250 of 0 to 4096 bytes 00000080:00200000:2.0:1787826221.276381:0:63537:0:(rw26.c:875:ll_write_end()) pos 1024000, len 4096, copied 4096 00000080:00200000:2.0:1787826221.276385:0:63537:0:(rw26.c:933:ll_write_end()) page@000000002ca65d01[3 0000000029ab925b 2 1 0000000008f41f32 0 0 250] 00000080:00200000:2.0:1787826221.276388:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000f33a6828 17ffffc0001001 3:0 ffff907da6246cb0 250 lru 00000080:00200000:2.0:1787826221.276392:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000ddad5343 250: 1< 0 - - > 2< 1024000 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 485711872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1787826221.276394:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000002ca65d01 00000080:00200000:2.0:1787826221.276395:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 251. 00000080:00200000:2.0:1787826221.276397:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 251 of 0 to 4096 bytes 00000080:00200000:2.0:1787826221.276423:0:63537:0:(rw26.c:875:ll_write_end()) pos 1028096, len 4096, copied 4096 00000080:00200000:2.0:1787826221.276428:0:63537:0:(rw26.c:933:ll_write_end()) page@0000000053668cc1[3 0000000029ab925b 2 1 0000000008f41f32 0 0 251] 00000080:00200000:2.0:1787826221.276430:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000000d05222b 17ffffc0001001 3:0 ffff907da6246bc8 251 lru 00000080:00200000:2.0:1787826221.276436:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000004ae548aa 251: 1< 0 - - > 2< 1028096 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 485711872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1787826221.276438:0:63537:0:(rw26.c:933:ll_write_end()) end page@0000000053668cc1 00000080:00200000:2.0:1787826221.276439:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 252. 00000080:00200000:2.0:1787826221.276441:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 252 of 0 to 4096 bytes 00000080:00200000:2.0:1787826221.276456:0:63537:0:(rw26.c:875:ll_write_end()) pos 1032192, len 4096, copied 4096 00000080:00200000:2.0:1787826221.276460:0:63537:0:(rw26.c:933:ll_write_end()) page@000000005fffcd13[3 0000000029ab925b 2 1 0000000008f41f32 0 0 252] 00000080:00200000:2.0:1787826221.276463:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @0000000065eb639a 17ffffc0001001 3:0 ffff907da6246740 252 lru 00000080:00200000:2.0:1787826221.276469:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000de29e63f 252: 1< 0 - - > 2< 1032192 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 485711872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1787826221.276471:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000005fffcd13 00000080:00200000:2.0:1787826221.276472:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 253. 00000080:00200000:2.0:1787826221.276474:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 253 of 0 to 4096 bytes 00000080:00200000:2.0:1787826221.276492:0:63537:0:(rw26.c:875:ll_write_end()) pos 1036288, len 4096, copied 4096 00000080:00200000:2.0:1787826221.276496:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000dbaaa24c[3 0000000029ab925b 2 1 0000000008f41f32 0 0 253] 00000080:00200000:2.0:1787826221.276499:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000129acb4b 17ffffc0001001 3:0 ffff907da6246488 253 lru 00000080:00200000:2.0:1787826221.276504:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@0000000001cea18d 253: 1< 0 - - > 2< 1036288 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 485711872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1787826221.276506:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000dbaaa24c 00000080:00200000:2.0:1787826221.276507:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 254. 00000080:00200000:2.0:1787826221.276509:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 254 of 0 to 4096 bytes 00000080:00200000:2.0:1787826221.276522:0:63537:0:(rw26.c:875:ll_write_end()) pos 1040384, len 4096, copied 4096 00000080:00200000:2.0:1787826221.276525:0:63537:0:(rw26.c:933:ll_write_end()) page@0000000012175948[3 0000000029ab925b 2 1 0000000008f41f32 0 0 254] 00000080:00200000:2.0:1787826221.276528:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000c8b7df7f 17ffffc0001001 3:0 ffff907da62463a0 254 lru 00000080:00200000:2.0:1787826221.276533:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000006e1b2829 254: 1< 0 - - > 2< 1040384 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 485711872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1787826221.276535:0:63537:0:(rw26.c:933:ll_write_end()) end page@0000000012175948 00000080:00200000:2.0:1787826221.276537:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 255. 00000080:00200000:2.0:1787826221.276538:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 255 of 0 to 4096 bytes 00000080:00200000:2.0:1787826221.276553:0:63537:0:(rw26.c:875:ll_write_end()) pos 1044480, len 4096, copied 4096 00000080:00200000:2.0:1787826221.276558:0:63537:0:(rw26.c:933:ll_write_end()) page@000000002da2f38a[3 0000000029ab925b 2 1 0000000008f41f32 0 0 255] 00000080:00200000:2.0:1787826221.276561:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @0000000077c4303e 17ffffc0001001 3:0 ffff907da6246e80 255 lru 00000080:00200000:2.0:1787826221.276566:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000dc1acad8 255: 1< 0 - - > 2< 1044480 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 485711872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1787826221.276568:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000002da2f38a 00000080:00200000:2.0:1787826221.276569:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 256. 00000080:00200000:2.0:1787826221.276573:0:63537:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 256, from 0, to 4096 prio 0 00000008:00010000:2.0:1787826221.277468:0:63537:0:(osc_cache.c:858:osc_extent_find()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.277497:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.277507:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.277516:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.277531:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.277541:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.277549:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.277559:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.277567:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.277575:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.277584:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.277603:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.279904:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.279917:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.279926:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1787826221.279948:0:63537:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d9221e578, count 15, dirtied 15 00000008:00010000:2.0:1787826221.279959:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.279968:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.279975:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.279983:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.279991:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.279999:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280013:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280021:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280029:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280037:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280045:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280055:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280062:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280070:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280077:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1787826221.280091:0:63537:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d9221e578, count 15, dirtied 15 00000008:00010000:2.0:1787826221.280096:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280105:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280113:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280121:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280129:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280137:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280145:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280152:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280166:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280174:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280182:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280190:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280199:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280207:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280216:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1787826221.280230:0:63537:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d9221e578, count 15, dirtied 15 00000008:00010000:2.0:1787826221.280235:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280255:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280262:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280271:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280279:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280287:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280296:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280306:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280314:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280322:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280331:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280340:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280348:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280357:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280365:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1787826221.280378:0:63537:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d9221e578, count 15, dirtied 15 00000008:00010000:2.0:1787826221.280384:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280393:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.280401:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282535:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282546:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282554:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282562:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282569:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282578:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282585:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282637:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282648:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282657:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282665:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282674:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1787826221.282692:0:63537:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d9221e578, count 15, dirtied 15 00000008:00010000:2.0:1787826221.282701:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282710:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282719:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282727:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282734:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282745:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282753:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282761:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282769:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282777:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282785:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282793:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282801:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282809:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282817:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1787826221.282849:0:63537:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d9221e578, count 15, dirtied 15 00000008:00010000:2.0:1787826221.282855:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282864:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282872:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282880:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282889:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282897:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282905:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282917:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282925:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282933:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282942:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282950:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282958:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282966:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.282973:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1787826221.282987:0:63537:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d9221e578, count 15, dirtied 15 00000008:00010000:2.0:1787826221.282993:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283002:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283010:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283018:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283027:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283035:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283044:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283052:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283060:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283073:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283081:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283089:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283097:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283106:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283114:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1787826221.283127:0:63537:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d9221e578, count 15, dirtied 15 00000008:00010000:2.0:1787826221.283133:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283142:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283156:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283164:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283173:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283181:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283190:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283198:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283207:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283215:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283223:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.283232:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285523:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285545:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285554:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1787826221.285571:0:63537:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d9221e578, count 15, dirtied 15 00000008:00010000:2.0:1787826221.285581:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285591:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285640:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285649:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285663:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285672:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285680:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285689:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285697:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285706:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285714:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285723:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285732:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285740:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285749:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1787826221.285764:0:63537:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d9221e578, count 15, dirtied 15 00000008:00010000:2.0:1787826221.285769:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285778:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285786:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285795:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285803:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285812:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285825:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285844:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285852:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285860:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285867:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285875:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285882:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285889:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285898:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1787826221.285911:0:63537:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d9221e578, count 15, dirtied 15 00000008:00010000:2.0:1787826221.285917:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285925:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285934:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285943:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285952:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285959:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285967:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285975:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285989:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.285998:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286006:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286015:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286021:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286030:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286038:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1787826221.286051:0:63537:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d9221e578, count 15, dirtied 15 00000008:00010000:2.0:1787826221.286056:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286069:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286078:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286087:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286096:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286105:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286114:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286122:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286129:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286137:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286144:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286152:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286160:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286169:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286177:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1787826221.286189:0:63537:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d9221e578, count 15, dirtied 15 00000008:00010000:2.0:1787826221.286195:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286203:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286211:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286224:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286232:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286241:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286791:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286800:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286809:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286817:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286825:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286843:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286851:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286859:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286868:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1787826221.286882:0:63537:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d9221e578, count 15, dirtied 15 00000008:00010000:2.0:1787826221.286889:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286896:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286903:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286911:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286919:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286933:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286942:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286951:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286960:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286969:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286977:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286986:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.286995:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287003:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287011:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1787826221.287023:0:63537:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d9221e578, count 15, dirtied 15 00000008:00010000:2.0:1787826221.287028:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287036:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287044:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287052:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287061:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287069:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287077:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287089:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287097:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287105:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287114:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287122:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287130:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287137:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287144:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1787826221.287156:0:63537:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d9221e578, count 15, dirtied 15 00000008:00010000:2.0:1787826221.287162:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287169:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287177:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287184:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287192:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287200:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287209:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287217:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287225:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287344:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287360:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287369:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287378:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287387:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826221.287395:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1787826221.287466:0:63537:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d9221e578, count 15, dirtied 15 00000008:00010000:2.0:1787826221.287475:0:63537:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1787826221.287486:0:63537:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d9221e578, count 1, dirtied 1 00000080:00200000:2.0:1787826221.287490:0:63537:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 256 pages 1048576 bytes, tot: 1048576 00000080:00200000:2.0:1787826221.287494:0:63537:0:(vvp_io.c:1421:vvp_io_write_start()) f200c.sanity-flr: write bytes 0, result: 1048576 00000020:00010000:2.0:1787826221.287511:0:63537:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da03eed58 (ffff907d83f04878/0) at cl_lock_release():245 00000020:00010000:2.0:1787826221.287515:0:63537:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da03eed58 (ffff907d83f04878/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826221.287519:0:63537:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c32db0 (ffff907d8662bb48/1) at cl_lock_cancel():142 00010000:00010000:2.0:1787826221.287525:0:63537:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826221.287537:0:63537:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1787826221.287553:0:63537:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da03eed58 (ffff907d83f04878/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826221.287556:0:63537:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c32db0 (ffff907d83f04878/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826221.287567:0:63537:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000080:00200000:2.0:1787826221.287574:0:63537:0:(file.c:2207:ll_file_io_generic()) Range unlock [0, 255] 00000080:00200000:2.0:1787826221.287585:0:63537:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f5:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826221.287588:0:63537:0:(file.c:2240:ll_file_io_generic()) f200c.sanity-flr: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1787826221.288577:0:63537:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f5:0x0] updating i_size 1048576 i_blocks 0 00000080:00200000:2.0:1787826221.288587:0:63537:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:3.0:1787826221.327971:0:63537:0:(file.c:2652:do_file_write_iter()) START file f200c.sanity-flr:[0x200000404:0x1f5:0x0], ppos: 1048576, count: 1048576 00000080:00200000:3.0:1787826221.327993:0:63537:0:(file.c:2090:ll_file_io_generic()) f200c.sanity-flr: write ppos: 1048576, bytes: 1048576 00000080:00200000:3.0:1787826221.327999:0:63537:0:(file.c:1848:ll_io_set_mirror()) f200c.sanity-flr: desiginated mirror: 0 00000020:00200000:3.0:1787826221.328009:0:63537:0:(cl_io.c:218:cl_io_rw_init()) header@000000000d663f42[0x110, 257, [0x200000404:0x1f5:0x0] hash] 00000020:00200000:3.0:1787826221.328012:0:63537:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1048576, 2097152) 0 0 00000080:00200000:3.0:1787826221.328018:0:63537:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f5:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787826221.328034:0:63537:0:(file.c:2170:ll_file_io_generic()) Range lock [256, 511] 00020000:00200000:3.0:1787826221.328043:0:63537:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 1048576 chunk: [1048576, 2097152) 2097152, 1048576 00020000:00200000:3.0:1787826221.328047:0:63537:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787826221.328066:0:63537:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [256, 511] 00000008:00010000:3.0:1787826221.328083:0:63537:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787826221.328088:0:63537:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d98a1c5b0, osc lock ffff907d963b83a8, flags 0x0 00010000:00010000:3.0:1787826221.328110:0:63537:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826221.328127:0:63537:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826221.328140:0:63537:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 7/0,2 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826221.328150:0:63537:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 7/0,2 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826221.328159:0:63537:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787826221.328170:0:63537:0:(vvp_io.c:1339:vvp_io_write_start()) f200c.sanity-flr: write [1048576, 2097152) append 0 append lockpos 0 00000080:00200000:3.0:1787826221.328181:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 256 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.328235:0:63537:0:(rw26.c:875:ll_write_end()) pos 1048576, len 4096, copied 4096 00000080:00200000:3.0:1787826221.329202:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000334fc4b8[3 0000000029ab925b 2 1 00000000854e1975 0 0 256] 00000080:00200000:3.0:1787826221.329207:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000006bec616c 17ffffc0001001 3:0 ffff907d9822f0e8 256 lru 00000080:00200000:3.0:1787826221.329215:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000003a4a0572 256: 1< 0 - - > 2< 1048576 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.329218:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000334fc4b8 00000080:00200000:3.0:1787826221.329219:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1787826221.329226:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 257 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.329324:0:63537:0:(rw26.c:875:ll_write_end()) pos 1052672, len 4096, copied 4096 00000080:00200000:3.0:1787826221.329329:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000f7cb8464[3 0000000029ab925b 2 1 00000000854e1975 0 0 257] 00000080:00200000:3.0:1787826221.329333:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @0000000026f8c73b 17ffffc0001001 3:0 ffff907d9822f658 257 lru 00000080:00200000:3.0:1787826221.329338:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000327d90b7 257: 1< 0 - - > 2< 1052672 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.329340:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000f7cb8464 00000080:00200000:3.0:1787826221.329341:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 2. 00000080:00200000:3.0:1787826221.329344:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 258 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.329359:0:63537:0:(rw26.c:875:ll_write_end()) pos 1056768, len 4096, copied 4096 00000080:00200000:3.0:1787826221.329364:0:63537:0:(rw26.c:933:ll_write_end()) page@000000005136285c[3 0000000029ab925b 2 1 00000000854e1975 0 0 258] 00000080:00200000:3.0:1787826221.329367:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @0000000021362889 17ffffc0001001 3:0 ffff907d9822f828 258 lru 00000080:00200000:3.0:1787826221.329372:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000a75e8586 258: 1< 0 - - > 2< 1056768 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.329374:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000005136285c 00000080:00200000:3.0:1787826221.329376:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 3. 00000080:00200000:3.0:1787826221.329377:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 259 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.329391:0:63537:0:(rw26.c:875:ll_write_end()) pos 1060864, len 4096, copied 4096 00000080:00200000:3.0:1787826221.329395:0:63537:0:(rw26.c:933:ll_write_end()) page@0000000035dce5e0[3 0000000029ab925b 2 1 00000000854e1975 0 0 259] 00000080:00200000:3.0:1787826221.329398:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000001c01a4ca 17ffffc0001001 3:0 ffff907d9822fd98 259 lru 00000080:00200000:3.0:1787826221.329469:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000d53a424c 259: 1< 0 - - > 2< 1060864 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.329473:0:63537:0:(rw26.c:933:ll_write_end()) end page@0000000035dce5e0 00000080:00200000:3.0:1787826221.329474:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 4. 00000080:00200000:3.0:1787826221.329476:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 260 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.329492:0:63537:0:(rw26.c:875:ll_write_end()) pos 1064960, len 4096, copied 4096 00000080:00200000:3.0:1787826221.329496:0:63537:0:(rw26.c:933:ll_write_end()) page@000000000d4c5ad3[3 0000000029ab925b 2 1 00000000854e1975 0 0 260] 00000080:00200000:3.0:1787826221.329499:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @0000000037165ede 17ffffc0001001 3:0 ffff907d9822f570 260 lru 00000080:00200000:3.0:1787826221.329504:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000005c1ab212 260: 1< 0 - - > 2< 1064960 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.329507:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000000d4c5ad3 00000080:00200000:3.0:1787826221.329508:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 5. 00000080:00200000:3.0:1787826221.329510:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 261 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.329521:0:63537:0:(rw26.c:875:ll_write_end()) pos 1069056, len 4096, copied 4096 00000080:00200000:3.0:1787826221.329525:0:63537:0:(rw26.c:933:ll_write_end()) page@000000004ebb0a3e[3 0000000029ab925b 2 1 00000000854e1975 0 0 261] 00000080:00200000:3.0:1787826221.329528:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000005f765d10 17ffffc0001001 3:0 ffff907d9822f9f8 261 lru 00000080:00200000:3.0:1787826221.329534:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000004c8bbb24 261: 1< 0 - - > 2< 1069056 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.329536:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000004ebb0a3e 00000080:00200000:3.0:1787826221.329537:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 6. 00000080:00200000:3.0:1787826221.329539:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 262 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.329549:0:63537:0:(rw26.c:875:ll_write_end()) pos 1073152, len 4096, copied 4096 00000080:00200000:3.0:1787826221.329553:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000744c69aa[3 0000000029ab925b 2 1 00000000854e1975 0 0 262] 00000080:00200000:3.0:1787826221.329556:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @0000000023dcaaed 17ffffc0001001 3:0 ffff907d9822fcb0 262 lru 00000080:00200000:3.0:1787826221.329561:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000ac9cef88 262: 1< 0 - - > 2< 1073152 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.329563:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000744c69aa 00000080:00200000:3.0:1787826221.329564:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 7. 00000080:00200000:3.0:1787826221.329566:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 263 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.329579:0:63537:0:(rw26.c:875:ll_write_end()) pos 1077248, len 4096, copied 4096 00000080:00200000:3.0:1787826221.329584:0:63537:0:(rw26.c:933:ll_write_end()) page@000000003b530e17[3 0000000029ab925b 2 1 00000000854e1975 0 0 263] 00000080:00200000:3.0:1787826221.329588:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000006dda13c8 17ffffc0001001 3:0 ffff907d9822fbc8 263 lru 00000080:00200000:3.0:1787826221.329592:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000001b70780d 263: 1< 0 - - > 2< 1077248 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.329594:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000003b530e17 00000080:00200000:3.0:1787826221.329595:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 8. 00000080:00200000:3.0:1787826221.329597:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 264 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.331394:0:63537:0:(rw26.c:875:ll_write_end()) pos 1081344, len 4096, copied 4096 00000080:00200000:3.0:1787826221.331418:0:63537:0:(rw26.c:933:ll_write_end()) page@000000000e96d8fd[3 0000000029ab925b 2 1 00000000854e1975 0 0 264] 00000080:00200000:3.0:1787826221.331421:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @0000000021526cdc 17ffffc0001021 2:0 ffff907d9822f740 264 lru 00000080:00200000:3.0:1787826221.331429:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000009d59539c 264: 1< 0 - - > 2< 1081344 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.331431:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000000e96d8fd 00000080:00200000:3.0:1787826221.331432:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 9. 00000080:00200000:3.0:1787826221.331437:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 265 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.331464:0:63537:0:(rw26.c:875:ll_write_end()) pos 1085440, len 4096, copied 4096 00000080:00200000:3.0:1787826221.331468:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000243a3ab2[3 0000000029ab925b 2 1 00000000854e1975 0 0 265] 00000080:00200000:3.0:1787826221.331470:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000be6cab35 17ffffc0001001 3:0 ffff907d9822f488 265 lru 00000080:00200000:3.0:1787826221.331475:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000001c3a5d2a 265: 1< 0 - - > 2< 1085440 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.331477:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000243a3ab2 00000080:00200000:3.0:1787826221.331478:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 10. 00000080:00200000:3.0:1787826221.331480:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 266 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.331495:0:63537:0:(rw26.c:875:ll_write_end()) pos 1089536, len 4096, copied 4096 00000080:00200000:3.0:1787826221.331500:0:63537:0:(rw26.c:933:ll_write_end()) page@000000000538d7a0[3 0000000029ab925b 2 1 00000000854e1975 0 0 266] 00000080:00200000:3.0:1787826221.331503:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000003e0d8d7b 17ffffc0001001 3:0 ffff907d9822f3a0 266 lru 00000080:00200000:3.0:1787826221.331507:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000c2c0e922 266: 1< 0 - - > 2< 1089536 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.331509:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000000538d7a0 00000080:00200000:3.0:1787826221.331510:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 11. 00000080:00200000:3.0:1787826221.331512:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 267 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.331527:0:63537:0:(rw26.c:875:ll_write_end()) pos 1093632, len 4096, copied 4096 00000080:00200000:3.0:1787826221.331531:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000eb1601de[3 0000000029ab925b 2 1 00000000854e1975 0 0 267] 00000080:00200000:3.0:1787826221.331533:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000eea807e6 17ffffc0001001 3:0 ffff907d9822fe80 267 lru 00000080:00200000:3.0:1787826221.331537:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000001fcd9cdf 267: 1< 0 - - > 2< 1093632 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.331541:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000eb1601de 00000080:00200000:3.0:1787826221.331542:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 12. 00000080:00200000:3.0:1787826221.331544:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 268 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.331559:0:63537:0:(rw26.c:875:ll_write_end()) pos 1097728, len 4096, copied 4096 00000080:00200000:3.0:1787826221.331563:0:63537:0:(rw26.c:933:ll_write_end()) page@0000000003fb22c1[3 0000000029ab925b 2 1 00000000854e1975 0 0 268] 00000080:00200000:3.0:1787826221.331566:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000d412f945 17ffffc0001001 3:0 ffff907d9822fae0 268 lru 00000080:00200000:3.0:1787826221.331571:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000e1e11da9 268: 1< 0 - - > 2< 1097728 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.331573:0:63537:0:(rw26.c:933:ll_write_end()) end page@0000000003fb22c1 00000080:00200000:3.0:1787826221.331574:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 13. 00000080:00200000:3.0:1787826221.331575:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 269 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.331590:0:63537:0:(rw26.c:875:ll_write_end()) pos 1101824, len 4096, copied 4096 00000080:00200000:3.0:1787826221.331594:0:63537:0:(rw26.c:933:ll_write_end()) page@0000000047312847[3 0000000029ab925b 2 1 00000000854e1975 0 0 269] 00000080:00200000:3.0:1787826221.331597:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000ab0ddd83 17ffffc0001001 3:0 ffff907d9822f000 269 lru 00000080:00200000:3.0:1787826221.331601:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000cc722978 269: 1< 0 - - > 2< 1101824 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.331603:0:63537:0:(rw26.c:933:ll_write_end()) end page@0000000047312847 00000080:00200000:3.0:1787826221.331604:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 14. 00000080:00200000:3.0:1787826221.332333:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 270 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.332360:0:63537:0:(rw26.c:875:ll_write_end()) pos 1105920, len 4096, copied 4096 00000080:00200000:3.0:1787826221.332366:0:63537:0:(rw26.c:933:ll_write_end()) page@0000000068f79f6c[3 0000000029ab925b 2 1 00000000854e1975 0 0 270] 00000080:00200000:3.0:1787826221.332369:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000991cc14f 17ffffc0001001 3:0 ffff907d9822f2b8 270 lru 00000080:00200000:3.0:1787826221.332374:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000944277d0 270: 1< 0 - - > 2< 1105920 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.332376:0:63537:0:(rw26.c:933:ll_write_end()) end page@0000000068f79f6c 00000080:00200000:3.0:1787826221.332377:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 15. 00000080:00200000:3.0:1787826221.332379:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 271 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.332394:0:63537:0:(rw26.c:875:ll_write_end()) pos 1110016, len 4096, copied 4096 00000080:00200000:3.0:1787826221.332399:0:63537:0:(rw26.c:933:ll_write_end()) page@0000000078872972[3 0000000029ab925b 2 1 00000000854e1975 0 0 271] 00000080:00200000:3.0:1787826221.332519:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000ef3a7647 17ffffc0001001 3:0 ffff907d9822f1d0 271 lru 00000080:00200000:3.0:1787826221.332525:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000daa94658 271: 1< 0 - - > 2< 1110016 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.332527:0:63537:0:(rw26.c:933:ll_write_end()) end page@0000000078872972 00000080:00200000:3.0:1787826221.332528:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 16. 00000080:00200000:3.0:1787826221.332531:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 272 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.332548:0:63537:0:(rw26.c:875:ll_write_end()) pos 1114112, len 4096, copied 4096 00000080:00200000:3.0:1787826221.332552:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000215a2478[3 0000000029ab925b 2 1 00000000854e1975 0 0 272] 00000080:00200000:3.0:1787826221.332555:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000f3cb3fd6 17ffffc0001001 3:0 ffff907d9822f910 272 lru 00000080:00200000:3.0:1787826221.332560:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000001c2cb302 272: 1< 0 - - > 2< 1114112 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.332562:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000215a2478 00000080:00200000:3.0:1787826221.332563:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 17. 00000080:00200000:3.0:1787826221.332565:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 273 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.332581:0:63537:0:(rw26.c:875:ll_write_end()) pos 1118208, len 4096, copied 4096 00000080:00200000:3.0:1787826221.332585:0:63537:0:(rw26.c:933:ll_write_end()) page@000000005408236f[3 0000000029ab925b 2 1 00000000854e1975 0 0 273] 00000080:00200000:3.0:1787826221.332588:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000000863377f 17ffffc0001001 3:0 ffff907d87fb0488 273 lru 00000080:00200000:3.0:1787826221.332593:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@0000000087c6b592 273: 1< 0 - - > 2< 1118208 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.332596:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000005408236f 00000080:00200000:3.0:1787826221.332597:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 18. 00000080:00200000:3.0:1787826221.332599:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 274 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.332938:0:63537:0:(rw26.c:875:ll_write_end()) pos 1122304, len 4096, copied 4096 00000080:00200000:3.0:1787826221.332944:0:63537:0:(rw26.c:933:ll_write_end()) page@0000000081b8c437[3 0000000029ab925b 2 1 00000000854e1975 0 0 274] 00000080:00200000:3.0:1787826221.332947:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @0000000006d12754 17ffffc0001001 3:0 ffff907d87fb0740 274 lru 00000080:00200000:3.0:1787826221.332952:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000002facb96f 274: 1< 0 - - > 2< 1122304 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.332954:0:63537:0:(rw26.c:933:ll_write_end()) end page@0000000081b8c437 00000080:00200000:3.0:1787826221.332955:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 19. 00000080:00200000:3.0:1787826221.332957:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 275 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.332972:0:63537:0:(rw26.c:875:ll_write_end()) pos 1126400, len 4096, copied 4096 00000080:00200000:3.0:1787826221.332976:0:63537:0:(rw26.c:933:ll_write_end()) page@0000000029ffede2[3 0000000029ab925b 2 1 00000000854e1975 0 0 275] 00000080:00200000:3.0:1787826221.332979:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000000a7a0da1 17ffffc0001001 3:0 ffff907d87fb0bc8 275 lru 00000080:00200000:3.0:1787826221.332983:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@0000000094266df8 275: 1< 0 - - > 2< 1126400 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.332985:0:63537:0:(rw26.c:933:ll_write_end()) end page@0000000029ffede2 00000080:00200000:3.0:1787826221.332986:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 20. 00000080:00200000:3.0:1787826221.332988:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 276 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.333004:0:63537:0:(rw26.c:875:ll_write_end()) pos 1130496, len 4096, copied 4096 00000080:00200000:3.0:1787826221.333007:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000381e5037[3 0000000029ab925b 2 1 00000000854e1975 0 0 276] 00000080:00200000:3.0:1787826221.333010:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000000b1bb61a 17ffffc0001001 3:0 ffff907d87fb0cb0 276 lru 00000080:00200000:3.0:1787826221.333015:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000d348b29f 276: 1< 0 - - > 2< 1130496 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.333016:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000381e5037 00000080:00200000:3.0:1787826221.333017:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 21. 00000080:00200000:3.0:1787826221.333019:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 277 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.333029:0:63537:0:(rw26.c:875:ll_write_end()) pos 1134592, len 4096, copied 4096 00000080:00200000:3.0:1787826221.333032:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000b7cbe767[3 0000000029ab925b 2 1 00000000854e1975 0 0 277] 00000080:00200000:3.0:1787826221.333035:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000008b91d7f0 17ffffc0001001 3:0 ffff907d87fb09f8 277 lru 00000080:00200000:3.0:1787826221.333039:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000f51b868a 277: 1< 0 - - > 2< 1134592 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.333041:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000b7cbe767 00000080:00200000:3.0:1787826221.333042:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 22. 00000080:00200000:3.0:1787826221.333043:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 278 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.333054:0:63537:0:(rw26.c:875:ll_write_end()) pos 1138688, len 4096, copied 4096 00000080:00200000:3.0:1787826221.333057:0:63537:0:(rw26.c:933:ll_write_end()) page@000000003e153342[3 0000000029ab925b 2 1 00000000854e1975 0 0 278] 00000080:00200000:3.0:1787826221.333060:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @0000000003138b97 17ffffc0001001 3:0 ffff907d87fb0570 278 lru 00000080:00200000:3.0:1787826221.333064:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000001645c1d5 278: 1< 0 - - > 2< 1138688 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.333066:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000003e153342 00000080:00200000:3.0:1787826221.333067:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 23. 00000080:00200000:3.0:1787826221.333068:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 279 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.333086:0:63537:0:(rw26.c:875:ll_write_end()) pos 1142784, len 4096, copied 4096 00000080:00200000:3.0:1787826221.333090:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000c95dd20b[3 0000000029ab925b 2 1 00000000854e1975 0 0 279] 00000080:00200000:3.0:1787826221.333093:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @0000000095c159be 17ffffc0001021 2:0 ffff907d87fb0828 279 lru 00000080:00200000:3.0:1787826221.333097:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000007a9092dc 279: 1< 0 - - > 2< 1142784 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.333100:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000c95dd20b 00000080:00200000:3.0:1787826221.333101:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 24. 00000080:00200000:3.0:1787826221.333102:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 280 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.333113:0:63537:0:(rw26.c:875:ll_write_end()) pos 1146880, len 4096, copied 4096 00000080:00200000:3.0:1787826221.333117:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000d9693d87[3 0000000029ab925b 2 1 00000000854e1975 0 0 280] 00000080:00200000:3.0:1787826221.333119:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000007ecf474f 17ffffc0001001 3:0 ffff907d87fb0d98 280 lru 00000080:00200000:3.0:1787826221.333124:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000110af44e 280: 1< 0 - - > 2< 1146880 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.333127:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000d9693d87 00000080:00200000:3.0:1787826221.333128:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 25. 00000080:00200000:3.0:1787826221.333130:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 281 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.333144:0:63537:0:(rw26.c:875:ll_write_end()) pos 1150976, len 4096, copied 4096 00000080:00200000:3.0:1787826221.333149:0:63537:0:(rw26.c:933:ll_write_end()) page@0000000069a8cb8a[3 0000000029ab925b 2 1 00000000854e1975 0 0 281] 00000080:00200000:3.0:1787826221.333151:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @0000000050e1d9d8 17ffffc0001001 3:0 ffff907d87fb0658 281 lru 00000080:00200000:3.0:1787826221.333157:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000003232285e 281: 1< 0 - - > 2< 1150976 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.333159:0:63537:0:(rw26.c:933:ll_write_end()) end page@0000000069a8cb8a 00000080:00200000:3.0:1787826221.333160:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 26. 00000080:00200000:3.0:1787826221.333162:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 282 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.333177:0:63537:0:(rw26.c:875:ll_write_end()) pos 1155072, len 4096, copied 4096 00000080:00200000:3.0:1787826221.333181:0:63537:0:(rw26.c:933:ll_write_end()) page@000000002847dce7[3 0000000029ab925b 2 1 00000000854e1975 0 0 282] 00000080:00200000:3.0:1787826221.333184:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000000034d5b7 17ffffc0001001 3:0 ffff907d87fb00e8 282 lru 00000080:00200000:3.0:1787826221.333189:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000df726859 282: 1< 0 - - > 2< 1155072 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.333191:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000002847dce7 00000080:00200000:3.0:1787826221.333192:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 27. 00000080:00200000:3.0:1787826221.333194:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 283 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.333209:0:63537:0:(rw26.c:875:ll_write_end()) pos 1159168, len 4096, copied 4096 00000080:00200000:3.0:1787826221.333214:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000d4a84516[3 0000000029ab925b 2 1 00000000854e1975 0 0 283] 00000080:00200000:3.0:1787826221.333217:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @0000000091b32fd1 17ffffc0001001 3:0 ffff907d87fb0910 283 lru 00000080:00200000:3.0:1787826221.333222:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000004f548fac 283: 1< 0 - - > 2< 1159168 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.333224:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000d4a84516 00000080:00200000:3.0:1787826221.333225:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 28. 00000080:00200000:3.0:1787826221.333227:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 284 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.333374:0:63537:0:(rw26.c:875:ll_write_end()) pos 1163264, len 4096, copied 4096 00000080:00200000:3.0:1787826221.333379:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000db3d0ee1[3 0000000029ab925b 2 1 00000000854e1975 0 0 284] 00000080:00200000:3.0:1787826221.333382:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000b2dabb64 17ffffc0001001 3:0 ffff907d87fb01d0 284 lru 00000080:00200000:3.0:1787826221.333387:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000756a6c89 284: 1< 0 - - > 2< 1163264 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.333389:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000db3d0ee1 00000080:00200000:3.0:1787826221.333390:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 29. 00000080:00200000:3.0:1787826221.333392:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 285 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.333479:0:63537:0:(rw26.c:875:ll_write_end()) pos 1167360, len 4096, copied 4096 00000080:00200000:3.0:1787826221.333484:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000aed7f510[3 0000000029ab925b 2 1 00000000854e1975 0 0 285] 00000080:00200000:3.0:1787826221.333487:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000e8f04ee1 17ffffc0001001 3:0 ffff907d87fb02b8 285 lru 00000080:00200000:3.0:1787826221.333493:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000ccb53b71 285: 1< 0 - - > 2< 1167360 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.333495:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000aed7f510 00000080:00200000:3.0:1787826221.333496:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 30. 00000080:00200000:3.0:1787826221.333498:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 286 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.333513:0:63537:0:(rw26.c:875:ll_write_end()) pos 1171456, len 4096, copied 4096 00000080:00200000:3.0:1787826221.333517:0:63537:0:(rw26.c:933:ll_write_end()) page@0000000099df4643[3 0000000029ab925b 2 1 00000000854e1975 0 0 286] 00000080:00200000:3.0:1787826221.333520:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000fdc2a7a5 17ffffc0001001 3:0 ffff907d87fb0000 286 lru 00000080:00200000:3.0:1787826221.333524:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000fec091c6 286: 1< 0 - - > 2< 1171456 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.333526:0:63537:0:(rw26.c:933:ll_write_end()) end page@0000000099df4643 00000080:00200000:3.0:1787826221.333527:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 31. 00000080:00200000:3.0:1787826221.333529:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 287 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.333543:0:63537:0:(rw26.c:875:ll_write_end()) pos 1175552, len 4096, copied 4096 00000080:00200000:3.0:1787826221.333547:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000e973e0e7[3 0000000029ab925b 2 1 00000000854e1975 0 0 287] 00000080:00200000:3.0:1787826221.333549:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000832c8b03 17ffffc0001001 3:0 ffff907d87fb0ae0 287 lru 00000080:00200000:3.0:1787826221.333555:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000004fb71673 287: 1< 0 - - > 2< 1175552 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.333557:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000e973e0e7 00000080:00200000:3.0:1787826221.333558:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 32. 00000080:00200000:3.0:1787826221.333559:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 288 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.333572:0:63537:0:(rw26.c:875:ll_write_end()) pos 1179648, len 4096, copied 4096 00000080:00200000:3.0:1787826221.333576:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000cf321c88[3 0000000029ab925b 2 1 00000000854e1975 0 0 288] 00000080:00200000:3.0:1787826221.333579:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000008e07d8b7 17ffffc0001001 3:0 ffff907d87fb0e80 288 lru 00000080:00200000:3.0:1787826221.333583:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000bd50b7c0 288: 1< 0 - - > 2< 1179648 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.333585:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000cf321c88 00000080:00200000:3.0:1787826221.333586:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 33. 00000080:00200000:3.0:1787826221.333587:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 289 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.333600:0:63537:0:(rw26.c:875:ll_write_end()) pos 1183744, len 4096, copied 4096 00000080:00200000:3.0:1787826221.333604:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000b082f8b9[3 0000000029ab925b 2 1 00000000854e1975 0 0 289] 00000080:00200000:3.0:1787826221.333876:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000005001fe07 17ffffc0001001 3:0 ffff907d87fb03a0 289 lru 00000080:00200000:3.0:1787826221.333882:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000b6f1d7df 289: 1< 0 - - > 2< 1183744 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.333885:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000b082f8b9 00000080:00200000:3.0:1787826221.333887:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 34. 00000080:00200000:3.0:1787826221.333889:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 290 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.333911:0:63537:0:(rw26.c:875:ll_write_end()) pos 1187840, len 4096, copied 4096 00000080:00200000:3.0:1787826221.333916:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000e4733bd1[3 0000000029ab925b 2 1 00000000854e1975 0 0 290] 00000080:00200000:3.0:1787826221.333918:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000003e01086d 17ffffc0001001 3:0 ffff907d835a49f8 290 lru 00000080:00200000:3.0:1787826221.333923:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000631955b9 290: 1< 0 - - > 2< 1187840 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.333924:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000e4733bd1 00000080:00200000:3.0:1787826221.333925:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 35. 00000080:00200000:3.0:1787826221.333927:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 291 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.333941:0:63537:0:(rw26.c:875:ll_write_end()) pos 1191936, len 4096, copied 4096 00000080:00200000:3.0:1787826221.333945:0:63537:0:(rw26.c:933:ll_write_end()) page@0000000024f12c55[3 0000000029ab925b 2 1 00000000854e1975 0 0 291] 00000080:00200000:3.0:1787826221.333947:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000007451e441 17ffffc0001001 3:0 ffff907d835a4570 291 lru 00000080:00200000:3.0:1787826221.333952:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000005ce27aed 291: 1< 0 - - > 2< 1191936 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.333954:0:63537:0:(rw26.c:933:ll_write_end()) end page@0000000024f12c55 00000080:00200000:3.0:1787826221.333955:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 36. 00000080:00200000:3.0:1787826221.333957:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 292 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.333970:0:63537:0:(rw26.c:875:ll_write_end()) pos 1196032, len 4096, copied 4096 00000080:00200000:3.0:1787826221.333974:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000a04cb709[3 0000000029ab925b 2 1 00000000854e1975 0 0 292] 00000080:00200000:3.0:1787826221.333976:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000008e8ddf91 17ffffc0001001 3:0 ffff907d835a4828 292 lru 00000080:00200000:3.0:1787826221.333981:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000ef1b3e89 292: 1< 0 - - > 2< 1196032 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.333983:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000a04cb709 00000080:00200000:3.0:1787826221.333984:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 37. 00000080:00200000:3.0:1787826221.333986:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 293 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.334000:0:63537:0:(rw26.c:875:ll_write_end()) pos 1200128, len 4096, copied 4096 00000080:00200000:3.0:1787826221.334003:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000a8782a37[3 0000000029ab925b 2 1 00000000854e1975 0 0 293] 00000080:00200000:3.0:1787826221.334005:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @0000000079b106eb 17ffffc0001001 3:0 ffff907d835a4d98 293 lru 00000080:00200000:3.0:1787826221.334009:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000c8bedb18 293: 1< 0 - - > 2< 1200128 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.334011:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000a8782a37 00000080:00200000:3.0:1787826221.334012:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 38. 00000080:00200000:3.0:1787826221.334014:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 294 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.334033:0:63537:0:(rw26.c:875:ll_write_end()) pos 1204224, len 4096, copied 4096 00000080:00200000:3.0:1787826221.334037:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000d7ee7de9[3 0000000029ab925b 2 1 00000000854e1975 0 0 294] 00000080:00200000:3.0:1787826221.334040:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000c2610461 17ffffc0001021 2:0 ffff907d835a4658 294 lru 00000080:00200000:3.0:1787826221.334045:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@0000000022e6ba85 294: 1< 0 - - > 2< 1204224 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.334047:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000d7ee7de9 00000080:00200000:3.0:1787826221.334048:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 39. 00000080:00200000:3.0:1787826221.334050:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 295 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.334063:0:63537:0:(rw26.c:875:ll_write_end()) pos 1208320, len 4096, copied 4096 00000080:00200000:3.0:1787826221.334067:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000d0a18867[3 0000000029ab925b 2 1 00000000854e1975 0 0 295] 00000080:00200000:3.0:1787826221.334070:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000008514d66c 17ffffc0001001 3:0 ffff907d835a40e8 295 lru 00000080:00200000:3.0:1787826221.334075:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000c270cc2c 295: 1< 0 - - > 2< 1208320 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.334077:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000d0a18867 00000080:00200000:3.0:1787826221.334078:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 40. 00000080:00200000:3.0:1787826221.334079:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 296 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.334094:0:63537:0:(rw26.c:875:ll_write_end()) pos 1212416, len 4096, copied 4096 00000080:00200000:3.0:1787826221.334098:0:63537:0:(rw26.c:933:ll_write_end()) page@000000003234f5cb[3 0000000029ab925b 2 1 00000000854e1975 0 0 296] 00000080:00200000:3.0:1787826221.334100:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000d79d226d 17ffffc0001001 3:0 ffff907d835a4910 296 lru 00000080:00200000:3.0:1787826221.334105:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@0000000029557ca7 296: 1< 0 - - > 2< 1212416 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.334107:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000003234f5cb 00000080:00200000:3.0:1787826221.334108:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 41. 00000080:00200000:3.0:1787826221.334110:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 297 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.334122:0:63537:0:(rw26.c:875:ll_write_end()) pos 1216512, len 4096, copied 4096 00000080:00200000:3.0:1787826221.334125:0:63537:0:(rw26.c:933:ll_write_end()) page@000000001f8ff535[3 0000000029ab925b 2 1 00000000854e1975 0 0 297] 00000080:00200000:3.0:1787826221.334128:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000003684a887 17ffffc0001001 3:0 ffff907d835a41d0 297 lru 00000080:00200000:3.0:1787826221.334132:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000cec3cf1b 297: 1< 0 - - > 2< 1216512 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.334134:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000001f8ff535 00000080:00200000:3.0:1787826221.334135:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 42. 00000080:00200000:3.0:1787826221.334137:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 298 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.334148:0:63537:0:(rw26.c:875:ll_write_end()) pos 1220608, len 4096, copied 4096 00000080:00200000:3.0:1787826221.334152:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000b0eb127d[3 0000000029ab925b 2 1 00000000854e1975 0 0 298] 00000080:00200000:3.0:1787826221.334154:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000007cbedc87 17ffffc0001001 3:0 ffff907d835a42b8 298 lru 00000080:00200000:3.0:1787826221.334159:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@0000000014109801 298: 1< 0 - - > 2< 1220608 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.334161:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000b0eb127d 00000080:00200000:3.0:1787826221.334162:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 43. 00000080:00200000:3.0:1787826221.334163:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 299 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.334174:0:63537:0:(rw26.c:875:ll_write_end()) pos 1224704, len 4096, copied 4096 00000080:00200000:3.0:1787826221.334179:0:63537:0:(rw26.c:933:ll_write_end()) page@0000000003b91c46[3 0000000029ab925b 2 1 00000000854e1975 0 0 299] 00000080:00200000:3.0:1787826221.334181:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000008a4ec62f 17ffffc0001001 3:0 ffff907d835a4000 299 lru 00000080:00200000:3.0:1787826221.334186:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000fd1651be 299: 1< 0 - - > 2< 1224704 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.334188:0:63537:0:(rw26.c:933:ll_write_end()) end page@0000000003b91c46 00000080:00200000:3.0:1787826221.334189:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 44. 00000080:00200000:3.0:1787826221.334191:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 300 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.334204:0:63537:0:(rw26.c:875:ll_write_end()) pos 1228800, len 4096, copied 4096 00000080:00200000:3.0:1787826221.334208:0:63537:0:(rw26.c:933:ll_write_end()) page@000000005ccc561c[3 0000000029ab925b 2 1 00000000854e1975 0 0 300] 00000080:00200000:3.0:1787826221.334211:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000006222dc64 17ffffc0001001 3:0 ffff907d835a4ae0 300 lru 00000080:00200000:3.0:1787826221.334216:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000004edcdffe 300: 1< 0 - - > 2< 1228800 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.334218:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000005ccc561c 00000080:00200000:3.0:1787826221.334219:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 45. 00000080:00200000:3.0:1787826221.334220:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 301 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.334233:0:63537:0:(rw26.c:875:ll_write_end()) pos 1232896, len 4096, copied 4096 00000080:00200000:3.0:1787826221.334511:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000db46fc85[3 0000000029ab925b 2 1 00000000854e1975 0 0 301] 00000080:00200000:3.0:1787826221.334515:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000009ef16016 17ffffc0001001 3:0 ffff907d835a4e80 301 lru 00000080:00200000:3.0:1787826221.334520:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@0000000084971eb3 301: 1< 0 - - > 2< 1232896 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.334522:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000db46fc85 00000080:00200000:3.0:1787826221.334523:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 46. 00000080:00200000:3.0:1787826221.334526:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 302 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.334543:0:63537:0:(rw26.c:875:ll_write_end()) pos 1236992, len 4096, copied 4096 00000080:00200000:3.0:1787826221.334548:0:63537:0:(rw26.c:933:ll_write_end()) page@0000000068634aea[3 0000000029ab925b 2 1 00000000854e1975 0 0 302] 00000080:00200000:3.0:1787826221.334551:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000ffc6aeab 17ffffc0001001 3:0 ffff907d835a43a0 302 lru 00000080:00200000:3.0:1787826221.334556:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000003372f4e1 302: 1< 0 - - > 2< 1236992 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.334557:0:63537:0:(rw26.c:933:ll_write_end()) end page@0000000068634aea 00000080:00200000:3.0:1787826221.334559:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 47. 00000080:00200000:3.0:1787826221.334561:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 303 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.334575:0:63537:0:(rw26.c:875:ll_write_end()) pos 1241088, len 4096, copied 4096 00000080:00200000:3.0:1787826221.334580:0:63537:0:(rw26.c:933:ll_write_end()) page@000000009a8eb7fa[3 0000000029ab925b 2 1 00000000854e1975 0 0 303] 00000080:00200000:3.0:1787826221.334582:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @0000000068f0ba8b 17ffffc0001001 3:0 ffff907d835a4488 303 lru 00000080:00200000:3.0:1787826221.334587:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000811d5531 303: 1< 0 - - > 2< 1241088 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.334589:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000009a8eb7fa 00000080:00200000:3.0:1787826221.334590:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 48. 00000080:00200000:3.0:1787826221.334592:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 304 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.334694:0:63537:0:(rw26.c:875:ll_write_end()) pos 1245184, len 4096, copied 4096 00000080:00200000:3.0:1787826221.334700:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000861378a9[3 0000000029ab925b 2 1 00000000854e1975 0 0 304] 00000080:00200000:3.0:1787826221.334703:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000fea00e61 17ffffc0001001 3:0 ffff907d835a4740 304 lru 00000080:00200000:3.0:1787826221.334709:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@0000000023c0a8b0 304: 1< 0 - - > 2< 1245184 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.334711:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000861378a9 00000080:00200000:3.0:1787826221.334712:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 49. 00000080:00200000:3.0:1787826221.334714:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 305 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.335034:0:63537:0:(rw26.c:875:ll_write_end()) pos 1249280, len 4096, copied 4096 00000080:00200000:3.0:1787826221.335040:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000ebe59a56[3 0000000029ab925b 2 1 00000000854e1975 0 0 305] 00000080:00200000:3.0:1787826221.335043:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @000000001e1f75d4 17ffffc0001001 3:0 ffff907d835a4bc8 305 lru 00000080:00200000:3.0:1787826221.335049:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000889e9350 305: 1< 0 - - > 2< 1249280 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.335051:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000ebe59a56 00000080:00200000:3.0:1787826221.335052:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 50. 00000080:00200000:3.0:1787826221.335054:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 306 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.335070:0:63537:0:(rw26.c:875:ll_write_end()) pos 1253376, len 4096, copied 4096 00000080:00200000:3.0:1787826221.335074:0:63537:0:(rw26.c:933:ll_write_end()) page@0000000044461282[3 0000000029ab925b 2 1 00000000854e1975 0 0 306] 00000080:00200000:3.0:1787826221.335076:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000f9c42f8d 17ffffc0001001 3:0 ffff907d835a4cb0 306 lru 00000080:00200000:3.0:1787826221.335081:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000008b280bf4 306: 1< 0 - - > 2< 1253376 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.335083:0:63537:0:(rw26.c:933:ll_write_end()) end page@0000000044461282 00000080:00200000:3.0:1787826221.335084:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 51. 00000080:00200000:3.0:1787826221.335085:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 307 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.335358:0:63537:0:(rw26.c:875:ll_write_end()) pos 1257472, len 4096, copied 4096 00000080:00200000:3.0:1787826221.335363:0:63537:0:(rw26.c:933:ll_write_end()) page@000000004d6962ec[3 0000000029ab925b 2 1 00000000854e1975 0 0 307] 00000080:00200000:3.0:1787826221.335367:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000118560c2 17ffffc0001001 3:0 ffff907d98a01740 307 lru 00000080:00200000:3.0:1787826221.335372:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@0000000083197786 307: 1< 0 - - > 2< 1257472 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.335375:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000004d6962ec 00000080:00200000:3.0:1787826221.335376:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 52. 00000080:00200000:3.0:1787826221.335378:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 308 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.335395:0:63537:0:(rw26.c:875:ll_write_end()) pos 1261568, len 4096, copied 4096 00000080:00200000:3.0:1787826221.335399:0:63537:0:(rw26.c:933:ll_write_end()) page@000000007e10c568[3 0000000029ab925b 2 1 00000000854e1975 0 0 308] 00000080:00200000:3.0:1787826221.335402:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @0000000014272625 17ffffc0001001 3:0 ffff907d98a01bc8 308 lru 00000080:00200000:3.0:1787826221.335474:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000a649d8fe 308: 1< 0 - - > 2< 1261568 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.335476:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000007e10c568 00000080:00200000:3.0:1787826221.335477:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 53. 00000080:00200000:3.0:1787826221.335479:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 309 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.335501:0:63537:0:(rw26.c:875:ll_write_end()) pos 1265664, len 4096, copied 4096 00000080:00200000:3.0:1787826221.335506:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000e857f86a[3 0000000029ab925b 2 1 00000000854e1975 0 0 309] 00000080:00200000:3.0:1787826221.335508:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @0000000063da7254 17ffffc0001021 2:0 ffff907d98a01cb0 309 lru 00000080:00200000:3.0:1787826221.335513:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000cddd686d 309: 1< 0 - - > 2< 1265664 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.335516:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000e857f86a 00000080:00200000:3.0:1787826221.335517:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 54. 00000080:00200000:3.0:1787826221.335519:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 310 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.335529:0:63537:0:(rw26.c:875:ll_write_end()) pos 1269760, len 4096, copied 4096 00000080:00200000:3.0:1787826221.335533:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000da622071[3 0000000029ab925b 2 1 00000000854e1975 0 0 310] 00000080:00200000:3.0:1787826221.335536:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000b1af5691 17ffffc0001001 3:0 ffff907d98a019f8 310 lru 00000080:00200000:3.0:1787826221.335541:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@000000008219f54a 310: 1< 0 - - > 2< 1269760 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.335543:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000da622071 00000080:00200000:3.0:1787826221.335544:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 55. 00000080:00200000:3.0:1787826221.335546:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 311 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.335558:0:63537:0:(rw26.c:875:ll_write_end()) pos 1273856, len 4096, copied 4096 00000080:00200000:3.0:1787826221.335562:0:63537:0:(rw26.c:933:ll_write_end()) page@00000000cf7f8fcd[3 0000000029ab925b 2 1 00000000854e1975 0 0 311] 00000080:00200000:3.0:1787826221.335564:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @00000000bab90367 17ffffc0001001 3:0 ffff907d98a01570 311 lru 00000080:00200000:3.0:1787826221.335570:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@0000000036a04252 311: 1< 0 - - > 2< 1273856 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.335574:0:63537:0:(rw26.c:933:ll_write_end()) end page@00000000cf7f8fcd 00000080:00200000:3.0:1787826221.335575:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 56. 00000080:00200000:3.0:1787826221.335577:0:63537:0:(rw26.c:681:ll_write_begin()) Writing 312 of 0 to 4096 bytes 00000080:00200000:3.0:1787826221.335592:0:63537:0:(rw26.c:875:ll_write_end()) pos 1277952, len 4096, copied 4096 00000080:00200000:3.0:1787826221.335596:0:63537:0:(rw26.c:933:ll_write_end()) page@000000000d1dbbb4[3 0000000029ab925b 2 1 00000000854e1975 0 0 312] 00000080:00200000:3.0:1787826221.335599:0:63537:0:(rw26.c:933:ll_write_end()) vmpage @0000000049b701c2 17ffffc0001001 3:0 ffff907d98a01828 312 lru 00000080:00200000:3.0:1787826221.335605:0:63537:0:(rw26.c:933:ll_write_end()) osc-page@00000000cfc00687 312: 1< 0 - - > 2< 1277952 0 0 0x0 0x0 | 000000003683d029 000000000134b611 > 3< 0 0 > 4< 0 0 8 484638720 - | - - + - > 5< - - + - | 0 - | 256 - -> 00000080:00200000:3.0:1787826221.336010:0:63537:0:(rw26.c:933:ll_write_end()) end page@000000000d1dbbb4 00000080:00200000:3.0:1787826221.336011:0:63537:0:(rw26.c:933:ll_write_end()) queued cl_page: 57. 00000080:00200000:2.0:1787826221.422112:0:63538:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826221.422139:0:63538:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826221.422201:0:63538:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8668b200/0xe5f736940357b3b8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 63538 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826221.422211:0:63538:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8668b200/0xe5f736940357b3b8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63538 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826221.422221:0:63538:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8668b200/0xe5f736940357b3b8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63538 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826221.422242:0:63538:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:63538:1874670011349888:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826221.428707:0:63538:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d866e9900 going to sleep for 11 seconds 00010000:00010000:3.0:1787826221.491669:0:63540:0:(ldlm_resource.c:195:lru_size_store()) dropping all unused locks from namespace lustre-MDT0000-mdc-ffff907d850a3000 00010000:00010000:3.0:1787826221.491724:0:63540:0:(ldlm_resource.c:195:lru_size_store()) dropping all unused locks from namespace lustre-MDT0000-mdc-ffff907d986d0000 00010000:00010000:3.0:1787826221.491937:0:63540:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 199/199 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1787826221.661108:0:63547:0:(ldlm_resource.c:195:lru_size_store()) dropping all unused locks from namespace lustre-OST0000-osc-ffff907d986d0000 00010000:00010000:2.0:1787826221.661158:0:63547:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 20/20 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1787826221.661171:0:60472:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 20 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1787826221.661179:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800028400000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:2.0:1787826221.661210:0:60472:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800029400000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:2.0:1787826221.661224:0:60472:0:(osc_cache.c:1174:osc_extent_make_ready()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800029400000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826221.661874:0:60472:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800029400000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00100000:2.0:1787826221.663071:0:60472:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d91149680 - object 0x240000400:797 offset 0<>2097152 00000100:00100000:2.0:1787826221.663147:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011351168:192.168.203.104@tcp:4:dd.0 00000008:00010000:3.0:1787826221.733086:0:2350:0:(osc_cache.c:908:osc_extent_finish()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800029400000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:3.0:1787826221.733954:0:2350:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000ec4f3a12 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800029400000000 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000100:00100000:3.0:1787826221.733992:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011351168:192.168.203.104@tcp:4:dd.0 00000080:00200000:2.0:1787826221.734377:0:60472:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f5:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1787826221.742393:0:60472:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f5:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1787826221.742444:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000ca4205d0/0xe5f736940357b268 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x31c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 33554432->18446744073709551615) gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0eabc expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826221.742466:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c633a10d/0xe5f736940357b25a lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x319:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0eaae expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826221.742484:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b317 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x318:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x28400010000 nid: local remote: 0x7918213f4fc0ec2f expref: -99 pid: 62921 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826221.742494:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b309 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x317:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x28400010000 nid: local remote: 0x7918213f4fc0ec28 expref: -99 pid: 62921 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826221.742504:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1dd66eb/0xe5f736940357b2fb lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x316:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x28400010000 nid: local remote: 0x7918213f4fc0ec21 expref: -99 pid: 62921 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826221.742514:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b2f4 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x312:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x28400010000 nid: local remote: 0x7918213f4fc0ec1a expref: -99 pid: 62921 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826221.742524:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e731006c/0xe5f736940357b2a7 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x31b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x28400020000 nid: local remote: 0x7918213f4fc0eb02 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826221.742535:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000888724c1/0xe5f736940357b237 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x31a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0ea7d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826221.742546:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c5159d29/0xe5f736940357b2e6 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x315:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x28400010000 nid: local remote: 0x7918213f4fc0ebbf expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826221.742555:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b55f8210/0xe5f736940357b2d1 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x314:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x28400010000 nid: local remote: 0x7918213f4fc0eba3 expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826221.742565:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000cd77128b/0xe5f7369403575a24 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x305:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x28400020000 nid: local remote: 0x7918213f4fc04c86 expref: -99 pid: 59743 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826221.742575:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000084f827e5/0xe5f7369403575a08 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc04c63 expref: -99 pid: 59743 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826221.742584:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000328eca16/0xe5f7369403571e88 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x2e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800028400020000 nid: local remote: 0x7918213f4fbfa77a expref: -99 pid: 58167 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00000080:00200000:2.0:1787826221.742644:0:60472:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1ae:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1787826221.742781:0:60472:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1ae:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1787826221.742795:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000026abc16/0xe5f7369403575872 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400010000 nid: local remote: 0x7918213f4fc0469e expref: -99 pid: 58975 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826221.742806:0:60472:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1eb:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1787826221.742811:0:60472:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1eb:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1787826221.742821:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000334a2bd8/0xe5f736940356a0b9 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028480000000 nid: local remote: 0x7918213f4fbe1d66 expref: -99 pid: 56204 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1787826221.742841:0:60472:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x6c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1787826221.742860:0:60472:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x6c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1787826221.742873:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000049c7d39e/0xe5f7369403569cfa lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x138:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x28400010000 nid: local remote: 0x7918213f4fbe11ff expref: -99 pid: 54412 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826221.742884:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000065bfc416/0xe5f7369403568bb2 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x12a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fbde23a expref: -99 pid: 52849 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826221.742900:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f7369403568acb lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x129:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->18446744073709551615) gid 0 flags: 0x800028400010000 nid: local remote: 0x7918213f4fbddfa8 expref: -99 pid: 52273 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826221.742911:0:60472:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x29:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1787826221.742916:0:60472:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x29:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1787826221.742926:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f7369403568ac4 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800028400010000 nid: local remote: 0x7918213f4fbddfa1 expref: -99 pid: 52273 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826221.742936:0:60472:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x29:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1787826221.742940:0:60472:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x29:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1787826221.742983:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b3b1 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x804869400000020 nid: local remote: 0x7918213f4fc0ed2b expref: -99 pid: 63537 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826221.742991:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000ca4205d0/0xe5f736940357b268 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x31c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 33554432->18446744073709551615) gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0eabc expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826221.742999:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c633a10d/0xe5f736940357b25a lrc: 1/0,0 mode: --/PW res: [0x240000400:0x319:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0eaae expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826221.743007:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b317 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x318:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x4829400010020 nid: local remote: 0x7918213f4fc0ec2f expref: -99 pid: 62921 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826221.743014:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b309 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x317:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x4829400010020 nid: local remote: 0x7918213f4fc0ec28 expref: -99 pid: 62921 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826221.743027:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1dd66eb/0xe5f736940357b2fb lrc: 1/0,0 mode: --/PR res: [0x240000400:0x316:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x4829400010020 nid: local remote: 0x7918213f4fc0ec21 expref: -99 pid: 62921 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826221.743034:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b2f4 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x312:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4829400010020 nid: local remote: 0x7918213f4fc0ec1a expref: -99 pid: 62921 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826221.743042:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e731006c/0xe5f736940357b2a7 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x31b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 16777216->18446744073709551615) gid 0 flags: 0x4829400020020 nid: local remote: 0x7918213f4fc0eb02 expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826221.743049:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000888724c1/0xe5f736940357b237 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x31a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 8388608->18446744073709551615) gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0ea7d expref: -99 pid: 62909 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826221.743056:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c5159d29/0xe5f736940357b2e6 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x315:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x4829400010020 nid: local remote: 0x7918213f4fc0ebbf expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826221.743062:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b55f8210/0xe5f736940357b2d1 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x314:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x4829400010020 nid: local remote: 0x7918213f4fc0eba3 expref: -99 pid: 62918 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826221.743069:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000cd77128b/0xe5f7369403575a24 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x305:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x4829400020020 nid: local remote: 0x7918213f4fc04c86 expref: -99 pid: 59743 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826221.743075:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000084f827e5/0xe5f7369403575a08 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc04c63 expref: -99 pid: 59743 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826221.743082:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000328eca16/0xe5f7369403571e88 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x2e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x804869400020020 nid: local remote: 0x7918213f4fbfa77a expref: -99 pid: 58167 timeout: 0 lvb_type: 1 lru_score: 12 lru_type: 0 00010000:00010000:2.0:1787826221.743094:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000026abc16/0xe5f7369403575872 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400010020 nid: local remote: 0x7918213f4fc0469e expref: -99 pid: 58975 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826221.743101:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000334a2bd8/0xe5f736940356a0b9 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x13c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869480000020 nid: local remote: 0x7918213f4fbe1d66 expref: -99 pid: 56204 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1787826221.743108:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000049c7d39e/0xe5f7369403569cfa lrc: 1/0,0 mode: --/PR res: [0x240000400:0x138:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4829400010020 nid: local remote: 0x7918213f4fbe11ff expref: -99 pid: 54412 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826221.743115:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000065bfc416/0xe5f7369403568bb2 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x12a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fbde23a expref: -99 pid: 52849 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826221.743122:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f7369403568acb lrc: 1/0,0 mode: --/PR res: [0x240000400:0x129:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->18446744073709551615) gid 0 flags: 0x804869400010020 nid: local remote: 0x7918213f4fbddfa8 expref: -99 pid: 52273 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826221.743129:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f7369403568ac4 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x128:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x804869400010020 nid: local remote: 0x7918213f4fbddfa1 expref: -99 pid: 52273 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826221.743135:0:60472:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 20 locks packed 00000100:00100000:2.0:1787826221.743161:0:60472:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_04:6f891bcb-15cd-41e0-96e2-db843619c1be:60472:1874670011351424:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1787826221.743248:0:60472:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d866e93c0 going to sleep for 11 seconds 00010000:00010000:2.0:1787826222.965220:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1787826223.099924:0:63559:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826223.099950:0:63559:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826223.100017:0:63559:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da70e4000/0xe5f736940357b3c6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 63559 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826223.100027:0:63559:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da70e4000/0xe5f736940357b3c6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63559 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826223.100036:0:63559:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da70e4000/0xe5f736940357b3c6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63559 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826223.100059:0:63559:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:63559:1874670011351552:192.168.203.104@tcp:101:dd.0 00000100:00100000:2.0:1787826223.100161:0:63559:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d866e9900 going to sleep for 11 seconds 00000001:02000400:2.0:1787826225.815772:0:63686:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: mirror extend 00000100:00100000:3.0:1787826225.872632:0:63688:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job lfs:d203faaf-7906-4d13-b1ba-8437467d6639:63688:1874670011351936:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826225.872657:0:63688:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826225.872671:0:63688:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826225.872683:0:63688:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826225.872696:0:63688:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826225.872708:0:63688:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826225.872717:0:63688:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826225.872724:0:63688:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826225.872736:0:63688:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d90847480 x1874670011351936/t0(0) o101->lustre-MDT0000-mdc-ffff907d850a3000@192.168.203.104@tcp:12/10 lens 576/688 e 0 to 0 dl 1787826241 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1787826225.872749:0:63688:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826225.872762:0:63688:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d9102d800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1787826225.872768:0:63688:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1787826225.872791:0:63688:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:3.0:1787826225.872807:0:63688:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d93310908) for lock 0xe5f736940357b3d4 00000080:00010000:3.0:1787826225.872813:0:63688:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d93310908) 00000080:00010000:3.0:1787826225.872815:0:63688:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b3d4 from it ffff9dd140e33bf8 00010000:00010000:3.0:1787826225.872819:0:63688:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826225.872833:0:63688:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787826225.872856:0:63688:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:3.0:1787826225.872882:0:63688:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f200c.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d93310908), file ffff907d82cb1500, open_flags 4d902, mode 0 opened 0 00000080:00200000:3.0:1787826225.872889:0:63688:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f200c.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d93310908), intent=open 00000002:00010000:3.0:1787826225.872910:0:63688:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f200c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 01154403 00010000:00010000:3.0:1787826225.872956:0:63688:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826225.872964:0:63688:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826225.872974:0:63688:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d850a3000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826225.872981:0:63688:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83526580 x1874670011352192/t0(0) o101->lustre-MDT0000-mdc-ffff907d850a3000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826225.872990:0:63688:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826225.873008:0:63688:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83526580 pname:cluuid:pid:xid:nid:opc:job lfs:d203faaf-7906-4d13-b1ba-8437467d6639:63688:1874670011352192:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826225.875939:0:63688:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d98acde40 going to sleep for 11 seconds 00000100:00100000:3.0:1787826225.929582:0:63688:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d850a3000: committing for last_committed 8589942867 gen 1 00000100:00100000:3.0:1787826225.929602:0:63688:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d83526580 x1874670011352192/t8589942867(8589942867) o101->lustre-MDT0000-mdc-ffff907d850a3000@192.168.203.104@tcp:12/10 lens 2240/664 e 0 to 0 dl 1787826241 ref 3 fl Rpc:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:3.0:1787826225.929645:0:63688:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83526580 pname:cluuid:pid:xid:nid:opc:job lfs:d203faaf-7906-4d13-b1ba-8437467d6639:63688:1874670011352192:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826225.929669:0:63688:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826225.929698:0:63688:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826225.929708:0:63688:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826225.929719:0:63688:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826225.929729:0:63688:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826225.929741:0:63688:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826225.929749:0:63688:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826225.929756:0:63688:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826225.929765:0:63688:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffff907d83526580 x1874670011352192/t8589942867(8589942867) o101->lustre-MDT0000-mdc-ffff907d850a3000@192.168.203.104@tcp:12/10 lens 2240/664 e 0 to 0 dl 1787826241 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1787826225.929778:0:63688:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d83526580 x1874670011352192/t8589942867(8589942867) o101->lustre-MDT0000-mdc-ffff907d850a3000@192.168.203.104@tcp:12/10 lens 2240/664 e 0 to 0 dl 1787826241 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1787826225.929791:0:63688:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1787826225.929813:0:63688:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826225.929823:0:63688:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d9102d800 type 13 mode 4 res 8589935620/501 (0 0) 00000080:00200000:3.0:1787826225.929829:0:63688:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1787826225.929857:0:63688:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000404:0x1f5:0x0](ffff907d93322a08) 00000080:00200000:3.0:1787826225.929928:0:63688:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1f5:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:3.0:1787826225.929934:0:63688:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f5:0x0](ffff907d93322a08) for lock 0xe5f736940357b3db 00000080:00010000:3.0:1787826225.929945:0:63688:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f5:0x0](ffff907d93322a08) 00000080:00200008:3.0:1787826225.929958:0:63688:0:(file.c:1090:ll_file_open()) START file f200c.sanity-flr:[0x200000404:0x1f5:0x0](ffff907d93322a08), flags 1154402 00000002:00100000:3.0:1787826225.929975:0:63688:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d83526580 x1874670011352192/t8589942867(8589942867) o101->lustre-MDT0000-mdc-ffff907d850a3000@192.168.203.104@tcp:12/10 lens 592/664 e 0 to 0 dl 1787826241 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:3.0:1787826225.930173:0:63688:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b3db from it ffff907da5717900 00010000:00010000:3.0:1787826225.930179:0:63688:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826225.930188:0:63688:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787826225.930208:0:63688:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1f5:0x0](ffff907d93322a08), inode mode 81a4 mask 46 00000080:00200080:3.0:1787826225.930236:0:63688:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1f5:0x0](ffff907d93322a08) cmd=800466af arg=7fffcbff519c 00000080:00200000:3.0:1787826225.930294:0:63688:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d93310908),name=/ 00000002:00010000:3.0:1787826225.930306:0:63688:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826225.930316:0:63688:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826225.930326:0:63688:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787826225.930333:0:63688:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826225.930337:0:63688:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d93310908) 00000080:00010000:3.0:1787826225.930340:0:63688:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d93310908) for lock 0xe5f736940357b3d4 00000080:00010000:3.0:1787826225.930344:0:63688:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b3d4 from it ffff9dd140e33bf8 00010000:00010000:3.0:1787826225.930347:0:63688:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826225.930355:0:63688:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787826225.930363:0:63688:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:3.0:1787826225.930378:0:63688:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d93310908),name=/ 00000002:00010000:3.0:1787826225.930383:0:63688:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826225.930389:0:63688:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826225.930396:0:63688:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787826225.930403:0:63688:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826225.930407:0:63688:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d93310908) 00000080:00010000:3.0:1787826225.930409:0:63688:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d93310908) for lock 0xe5f736940357b3d4 00000080:00010000:3.0:1787826225.930413:0:63688:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b3d4 from it ffff9dd140e33b38 00010000:00010000:3.0:1787826225.930415:0:63688:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826225.930422:0:63688:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00000080:00200000:3.0:1787826225.930430:0:63688:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d93310908), inode mode 41ed mask 3 00000080:00200000:3.0:1787826225.930434:0:63688:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=. :VOLATILE:0000:5CE74C01:fd=03, dir=[0x200000007:0x1:0x0](ffff907d93310908), file ffff907d88b61500, open_flags 2f9c1, mode 8180 opened 0 00000080:00200000:3.0:1787826225.930440:0:63688:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=. :VOLATILE:0000:5CE74C01:fd=03, dir=[0x200000007:0x1:0x0](ffff907d93310908), intent=open|creat 00000100:00100000:3.0:1787826225.930482:0:63688:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job lfs:d203faaf-7906-4d13-b1ba-8437467d6639:63688:1874670011353088:192.168.203.104@tcp:700:lfs.0 00000100:00100000:3.0:1787826225.930599:0:63688:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d98acdb40 going to sleep for 11 seconds 00000100:00100000:3.0:1787826225.943336:0:63688:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job lfs:d203faaf-7906-4d13-b1ba-8437467d6639:63688:1874670011353088:192.168.203.104@tcp:700:lfs.0 00000002:00010000:3.0:1787826225.943368:0:63688:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: . :VOLATILE:0000:5CE74C01:fd=03,[0x200000bd1:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0574702 00010000:00010000:3.0:1787826225.943388:0:63688:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00010000:3.0:1787826225.943442:0:63688:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00010000:3.0:1787826225.943450:0:63688:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787826225.943454:0:63688:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a800/0xe5f736940357b3d4 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0ed7f expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00010000:3.0:1787826225.943497:0:63688:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86042c00/0xe5f736940357b3e2 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826225.943509:0:63688:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86042c00/0xe5f736940357b3e2 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826225.943520:0:63688:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d850a3000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826225.943529:0:63688:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d835252c0 x1874670011353344/t0(0) o101->lustre-MDT0000-mdc-ffff907d850a3000@192.168.203.104@tcp:12/10 lens 552/67896 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826225.943538:0:63688:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86042c00/0xe5f736940357b3e2 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826225.943557:0:63688:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job lfs:d203faaf-7906-4d13-b1ba-8437467d6639:63688:1874670011353344:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826225.943700:0:63688:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d98acdb40 going to sleep for 11 seconds 00000100:00100000:2.0:1787826225.971893:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826225.972014:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025467264 00000100:00100000:3.0:1787826225.972023:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025467264 00000100:00100000:3.0:1787826225.972079:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7088 00000100:00100000:3.0:1787826225.972089:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d9114b840 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:d203faaf-7906-4d13-b1ba-8437467d6639+5:32458:x1874670025467264:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826225.972104:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86042c00/0xe5f736940357b3e2 lrc: 2/0,0 mode: PR/PR res: [0x200000bd1:0x1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0ee3c expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826225.972179:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826225.972186:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d9114b840 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:d203faaf-7906-4d13-b1ba-8437467d6639+5:32458:x1874670025467264:12345-192.168.203.104@tcp:104: Request processed in 99us (300us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826225.972193:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7088 00000100:00100000:2.0:1787826225.991775:0:63688:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job lfs:d203faaf-7906-4d13-b1ba-8437467d6639:63688:1874670011353600:192.168.203.104@tcp:101:lfs.0 00010000:00010000:2.0:1787826225.991810:0:63688:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d83f0ae00/0xe5f736940357b3e9 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826225.991825:0:63688:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d83f0ae00/0xe5f736940357b3e9 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826225.991832:0:63688:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d83f0ae00/0xe5f736940357b3e9 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826225.991840:0:63688:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d83f0ae00/0xe5f736940357b3e9 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826225.991848:0:63688:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d83f0ae00/0xe5f736940357b3e9 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826225.991854:0:63688:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0ae00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:2.0:1787826225.991859:0:63688:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d83f0ae00/0xe5f736940357b3e9 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826225.991870:0:63688:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d83f0ae00/0xe5f736940357b3e9 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826225.991877:0:63688:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d83f0ae00/0xe5f736940357b3e9 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826225.991884:0:63688:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1787826225.991891:0:63688:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d83f0ae00/0xe5f736940357b3e9 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826225.991897:0:63688:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d83f0ae00/0xe5f736940357b3e9 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826225.991923:0:63688:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x800002b, status=0 req@ffff907d908a2580 x1874670011353600/t8589942869(8589942869) o101->lustre-MDT0000-mdc-ffff907d850a3000@192.168.203.104@tcp:12/10 lens 2224/768 e 0 to 0 dl 1787826241 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1787826225.991936:0:63688:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a2580 x1874670011353600/t8589942869(8589942869) o101->lustre-MDT0000-mdc-ffff907d850a3000@192.168.203.104@tcp:12/10 lens 2224/768 e 0 to 0 dl 1787826241 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1787826225.991952:0:63688:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1787826225.991992:0:63688:0:(vvp_io.c:1889:vvp_io_init()) [0x200000bd1:0x1:0x0] ignore/verify layout 1/0, layout version 29 restore needed 0 00000080:00200000:2.0:1787826225.992001:0:63688:0:(vvp_io.c:304:vvp_io_fini()) [0x200000bd1:0x1:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826225.992043:0:63688:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000bd1:0x1:0x0], updating i_size 0 00000002:00100000:2.0:1787826225.992060:0:63688:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a2580 x1874670011353600/t8589942869(8589942869) o101->lustre-MDT0000-mdc-ffff907d850a3000@192.168.203.104@tcp:12/10 lens 680/768 e 0 to 0 dl 1787826241 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1787826225.992097:0:63688:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d908a2580 x1874670011353600/t8589942869(8589942869) o101->lustre-MDT0000-mdc-ffff907d850a3000@192.168.203.104@tcp:12/10 lens 680/768 e 0 to 0 dl 1787826241 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1787826225.992107:0:63688:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d850a3000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1787826225.992113:0:63688:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114a580 x1874670011353984/t0(0) o35->lustre-MDT0000-mdc-ffff907d850a3000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1787826225.992136:0:63688:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job lfs:d203faaf-7906-4d13-b1ba-8437467d6639:63688:1874670011353984:192.168.203.104@tcp:35:lfs.0 00000100:00100000:2.0:1787826225.992205:0:63688:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d866e96c0 going to sleep for 11 seconds 00000100:00100000:2.0:1787826226.018949:0:63688:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job lfs:d203faaf-7906-4d13-b1ba-8437467d6639:63688:1874670011353984:192.168.203.104@tcp:35:lfs.0 00000002:00100000:2.0:1787826226.018985:0:63688:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d908a2580 x1874670011353600/t8589942869(8589942869) o101->lustre-MDT0000-mdc-ffff907d850a3000@192.168.203.104@tcp:12/10 lens 680/768 e 0 to 0 dl 1787826241 ref 2 fl Complete:RQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1787826226.019032:0:63688:0:(file.c:6313:ll_getattr_dentry()) START file f200c.sanity-flr:[0x200000404:0x1f5:0x0](ffff907d93322a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1787826226.019041:0:63688:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1f5:0x0](ffff907d93322a08),name=f200c.sanity-flr 00000002:00010000:2.0:1787826226.019062:0:63688:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f5:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826226.019090:0:63688:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826226.019103:0:63688:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826226.019112:0:63688:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826226.019117:0:63688:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f5:0x0](ffff907d93322a08) 00000080:00010000:2.0:1787826226.019121:0:63688:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f5:0x0](ffff907d93322a08) for lock 0xe5f736940357b3db 00000080:00010000:2.0:1787826226.019128:0:63688:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b3db from it ffff9dd140e33cd8 00010000:00010000:2.0:1787826226.019132:0:63688:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826226.019140:0:63688:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1787826226.019166:0:63688:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f5:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1787826226.019178:0:63688:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1f5:0x0] 00000008:00010000:2.0:1787826226.019196:0:63688:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826226.019199:0:63688:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322bb0, osc lock ffff907d830a2d68, flags 0x400001002 00010000:00010000:2.0:1787826226.019240:0:63688:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.019250:0:63688:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1787826226.019279:0:63688:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:2.0:1787826226.021474:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:d203faaf-7906-4d13-b1ba-8437467d6639:2351:1874670011354112:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826226.029155:0:2351:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.029182:0:2351:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.029196:0:2351:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.029223:0:2351:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:3.0:1787826226.029239:0:2351:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:3.0:1787826226.029250:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:3.0:1787826226.029262:0:2351:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.029270:0:2351:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.029278:0:2351:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.029287:0:2351:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1787826226.029306:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:d203faaf-7906-4d13-b1ba-8437467d6639:2351:1874670011354112:192.168.203.104@tcp:101:lfs.0 00000080:00200000:2.0:1787826226.029865:0:63688:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f5:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826226.029872:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d83f02780 (ffff907d83f04878/0) at cl_lock_release():245 00000020:00010000:2.0:1787826226.029876:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d83f02780 (ffff907d83f04878/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826226.029880:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322bb0 (ffff907d8662bb48/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826226.029885:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d83f02780 (ffff907d83f04878/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826226.029887:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322bb0 (ffff907d83f04878/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826226.029901:0:63688:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f5:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826226.029919:0:63688:0:(file.c:6313:ll_getattr_dentry()) START file . :VOLATILE:0000:5CE74C01:fd=03:[0x200000bd1:0x1:0x0](ffff907d92219148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1787826226.029927:0:63688:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000bd1:0x1:0x0](ffff907d92219148),name=. :VOLATILE:0000:5CE74C01:fd=03 00000002:00010000:2.0:1787826226.029942:0:63688:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000bd1:0x1:0x0]) in obj [0x200000bd1:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826226.029987:0:63688:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907da70e5a00/0xe5f736940357b3f7 lrc: 3/1,0 mode: --/CR res: [0x200000bd1:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.029996:0:63688:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907da70e5a00/0xe5f736940357b3f7 lrc: 3/1,0 mode: --/CR res: [0x200000bd1:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.030006:0:63688:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907da70e5a00/0xe5f736940357b3f7 lrc: 3/1,0 mode: --/CR res: [0x200000bd1:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826226.030027:0:63688:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job lfs:d203faaf-7906-4d13-b1ba-8437467d6639:63688:1874670011354240:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826226.031434:0:63688:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d866e96c0 going to sleep for 11 seconds 00010000:00010000:2.0:1787826226.039140:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826226.041861:0:63688:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job lfs:d203faaf-7906-4d13-b1ba-8437467d6639:63688:1874670011354240:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826226.041888:0:63688:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907da70e5a00/0xe5f736940357b3f7 lrc: 4/1,0 mode: --/CR res: [0x200000bd1:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ee74 expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.041900:0:63688:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907da70e5a00/0xe5f736940357b3f7 lrc: 4/1,0 mode: --/PR res: [0x200000bd1:0x1:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ee74 expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.041912:0:63688:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907da70e5a00/0xe5f736940357b3f7 lrc: 4/1,0 mode: PR/PR res: [0x200000bd1:0x1:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ee74 expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.041922:0:63688:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907da70e5a00/0xe5f736940357b3f7 lrc: 4/1,0 mode: PR/PR res: [0x200000bd1:0x1:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ee74 expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.041934:0:63688:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907da70e5a00/0xe5f736940357b3f7 lrc: 5/2,0 mode: PR/PR res: [0x200000bd1:0x1:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ee74 expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.041942:0:63688:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907da70e5a00/0xe5f736940357b3f7 lrc: 5/2,0 mode: PR/PR res: [0x200000bd1:0x1:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ee74 expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.041950:0:63688:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907da70e5a00/0xe5f736940357b3f7 lrc: 4/1,0 mode: PR/PR res: [0x200000bd1:0x1:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ee74 expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826226.041960:0:63688:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d9114a940 x1874670011354240/t0(0) o101->lustre-MDT0000-mdc-ffff907d850a3000@192.168.203.104@tcp:12/10 lens 576/744 e 0 to 0 dl 1787826242 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1787826226.041970:0:63688:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 160 ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907da70e5a00/0xe5f736940357b3f7 lrc: 3/1,0 mode: PR/PR res: [0x200000bd1:0x1:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ee74 expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1787826226.041987:0:63688:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907da70e5a00/0xe5f736940357b3f7 lrc: 3/1,0 mode: PR/PR res: [0x200000bd1:0x1:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ee74 expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.041999:0:63688:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d9102d800 type 13 mode 4 res 8589937617/1 (0 0) 00000080:00200000:3.0:1787826226.042006:0:63688:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000bd1:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1787826226.042059:0:63688:0:(file.c:6854:ll_layout_conf()) [0x200000bd1:0x1:0x0]: layout version change: 4294967294 -> 1 00000080:00010000:3.0:1787826226.042064:0:63688:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000bd1:0x1:0x0](ffff907d92219148) for lock 0xe5f736940357b3f7 00000080:00010000:3.0:1787826226.042069:0:63688:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000bd1:0x1:0x0](ffff907d92219148) 00000080:00010000:3.0:1787826226.042072:0:63688:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b3f7 from it ffff9dd140e33cd8 00010000:00010000:3.0:1787826226.042075:0:63688:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907da70e5a00/0xe5f736940357b3f7 lrc: 3/1,0 mode: PR/PR res: [0x200000bd1:0x1:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0ee74 expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.042084:0:63688:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907da70e5a00/0xe5f736940357b3f7 lrc: 2/0,0 mode: PR/PR res: [0x200000bd1:0x1:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0ee74 expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787826226.042115:0:63688:0:(vvp_io.c:1889:vvp_io_init()) [0x200000bd1:0x1:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1787826226.042125:0:63688:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000bd1:0x1:0x0] 00000008:00010000:3.0:1787826226.042142:0:63688:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826226.042145:0:63688:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d98acd7b0, osc lock ffff907d963b94b8, flags 0x400001002 00000008:00010000:3.0:1787826226.042152:0:63688:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826226.042153:0:63688:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d98acd7f0, osc lock ffff907d963b9c08, flags 0x400001002 00010000:00010000:3.0:1787826226.042195:0:63688:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d850a3000 lock: 0000000088389957/0xe5f736940357b3fe lrc: 3/1,0 mode: --/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.043416:0:63688:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff907d850a3000 lock: 0000000088389957/0xe5f736940357b3fe lrc: 3/1,0 mode: --/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.043510:0:63688:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d850a3000 lock: 0000000029c7e506/0xe5f736940357b405 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826226.043578:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83526940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:d203faaf-7906-4d13-b1ba-8437467d6639:2350:1874670011354368:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826226.044404:0:63688:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff907d850a3000 lock: 0000000029c7e506/0xe5f736940357b405 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:3.0:1787826226.044433:0:63688:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000100:00100000:3.0:1787826226.044457:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:d203faaf-7906-4d13-b1ba-8437467d6639:2351:1874670011354496:192.168.203.104@tcp:101:lfs.0 00010000:00010000:2.0:1787826226.060864:0:2351:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d850a3000 lock: 0000000029c7e506/0xe5f736940357b405 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.060864:0:2350:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d850a3000 lock: 0000000088389957/0xe5f736940357b3fe lrc: 6/2,0 mode: --/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.060890:0:2351:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff907d850a3000 lock: 0000000029c7e506/0xe5f736940357b405 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ee7b expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.060890:0:2350:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d850a3000 lock: 0000000088389957/0xe5f736940357b3fe lrc: 6/2,0 mode: --/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ee82 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.060906:0:2351:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d850a3000 lock: 0000000029c7e506/0xe5f736940357b405 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ee7b expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.060906:0:2350:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d850a3000 lock: 0000000088389957/0xe5f736940357b3fe lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ee82 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.060917:0:2350:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d850a3000 lock: 0000000088389957/0xe5f736940357b3fe lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ee82 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.060920:0:2351:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff907d850a3000 lock: 0000000029c7e506/0xe5f736940357b405 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ee7b expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:3.0:1787826226.060932:0:2350:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d850a3000 lock: 0000000088389957/0xe5f736940357b3fe lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0ee82 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:3.0:1787826226.060943:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826226.060949:0:2350:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d850a3000 lock: 0000000088389957/0xe5f736940357b3fe lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ee82 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.060956:0:2350:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d850a3000 lock: 0000000088389957/0xe5f736940357b3fe lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ee82 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.060964:0:2350:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d850a3000 lock: 0000000088389957/0xe5f736940357b3fe lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ee82 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.060972:0:2350:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d850a3000 lock: 0000000088389957/0xe5f736940357b3fe lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ee82 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1787826226.060990:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83526940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:d203faaf-7906-4d13-b1ba-8437467d6639:2350:1874670011354368:192.168.203.104@tcp:101:lfs.0 00000008:00010000:2.0:1787826226.061046:0:2351:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff907d850a3000 lock: 0000000029c7e506/0xe5f736940357b405 lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0ee7b expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1787826226.061056:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:2.0:1787826226.061070:0:2351:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d850a3000 lock: 0000000029c7e506/0xe5f736940357b405 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ee7b expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.061078:0:2351:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d850a3000 lock: 0000000029c7e506/0xe5f736940357b405 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ee7b expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.061086:0:2351:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d850a3000 lock: 0000000029c7e506/0xe5f736940357b405 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ee7b expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.061094:0:2351:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d850a3000 lock: 0000000029c7e506/0xe5f736940357b405 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ee7b expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:2.0:1787826226.061106:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:d203faaf-7906-4d13-b1ba-8437467d6639:2351:1874670011354496:192.168.203.104@tcp:101:lfs.0 00000080:00200000:3.0:1787826226.061304:0:63688:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000bd1:0x1:0x0] updating i_size 0 i_blocks 0 00000020:00010000:3.0:1787826226.061310:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da61e98b0 (ffff907d8662b1b8/0) at cl_lock_release():245 00000020:00010000:3.0:1787826226.061314:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da61e98b0 (ffff907d8662b1b8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826226.061319:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98acd7b0 (ffff907da5608128/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826226.061324:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98acd7f0 (ffff907da5608368/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826226.061326:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da61e98b0 (ffff907d8662b1b8/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826226.061329:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98acd7b0 (ffff907d8662b1b8/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826226.061333:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98acd7f0 (ffff907d8662b1b8/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826226.061345:0:63688:0:(vvp_io.c:304:vvp_io_fini()) [0x200000bd1:0x1:0x0] ignore/verify layout 0/1, layout version 1 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:3.0:1787826226.061372:0:63688:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1f5:0x0](ffff907d93322a08) cmd=c00c66f3 arg=7fffcbff72e4 00000002:00010000:3.0:1787826226.061394:0:63688:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f5:0x0]) in obj [0x200000404:0x1f5:0x0], intent: open flags 01004040000001 00010000:00010000:3.0:1787826226.061454:0:63688:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86042600/0xe5f736940357b40c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.061463:0:63688:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x81000000001000 ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86042600/0xe5f736940357b40c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826226.061474:0:63688:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d850a3000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826226.061482:0:63688:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83526940 x1874670011354752/t0(0) o101->lustre-MDT0000-mdc-ffff907d850a3000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826226.061493:0:63688:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86042600/0xe5f736940357b40c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x81000000000000 pid: 63688 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826226.061511:0:63688:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83526940 pname:cluuid:pid:xid:nid:opc:job lfs:d203faaf-7906-4d13-b1ba-8437467d6639:63688:1874670011354752:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826226.061698:0:63688:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d98acd780 going to sleep for 11 seconds 00000100:00100000:3.0:1787826226.073020:0:63688:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83526940 pname:cluuid:pid:xid:nid:opc:job lfs:d203faaf-7906-4d13-b1ba-8437467d6639:63688:1874670011354752:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826226.073044:0:63688:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PW ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86042600/0xe5f736940357b40c lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc0ee90 expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.073056:0:63688:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86042600/0xe5f736940357b40c lrc: 4/1,0 mode: --/PW res: [0x200000404:0x1f5:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc0ee90 expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.073068:0:63688:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86042600/0xe5f736940357b40c lrc: 4/1,0 mode: PW/PW res: [0x200000404:0x1f5:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0ee90 expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.073079:0:63688:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86042600/0xe5f736940357b40c lrc: 4/1,0 mode: PW/PW res: [0x200000404:0x1f5:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc0ee90 expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.073092:0:63688:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86042600/0xe5f736940357b40c lrc: 5/1,1 mode: PW/PW res: [0x200000404:0x1f5:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0ee90 expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.073105:0:63688:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86042600/0xe5f736940357b40c lrc: 5/1,1 mode: PW/PW res: [0x200000404:0x1f5:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0ee90 expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.073112:0:63688:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86042600/0xe5f736940357b40c lrc: 4/0,1 mode: PW/PW res: [0x200000404:0x1f5:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0ee90 expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826226.073122:0:63688:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x600002b, status=0 req@ffff907d83526940 x1874670011354752/t8589942871(8589942871) o101->lustre-MDT0000-mdc-ffff907d850a3000@192.168.203.104@tcp:12/10 lens 2224/664 e 0 to 0 dl 1787826242 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1787826226.073134:0:63688:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d83526940 x1874670011354752/t8589942871(8589942871) o101->lustre-MDT0000-mdc-ffff907d850a3000@192.168.203.104@tcp:12/10 lens 2224/664 e 0 to 0 dl 1787826242 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1787826226.073148:0:63688:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86042600/0xe5f736940357b40c lrc: 3/0,1 mode: PW/PW res: [0x200000404:0x1f5:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc0ee90 expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.073159:0:63688:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d9102d800 type 13 mode 2 res 8589935620/501 (0 0) 00000080:00200000:3.0:1787826226.073165:0:63688:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1f5:0x0] suppgids 0 0: rc 0 00000002:00100000:3.0:1787826226.073180:0:63688:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d83526940 x1874670011354752/t8589942871(8589942871) o101->lustre-MDT0000-mdc-ffff907d850a3000@192.168.203.104@tcp:12/10 lens 576/664 e 0 to 0 dl 1787826242 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:3.0:1787826226.073187:0:63688:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f5:0x0](ffff907d93322a08) for lock 0xe5f736940357b40c 00000080:00010000:3.0:1787826226.073192:0:63688:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b40c from it ffff9dd140e33cc0 00010000:00010000:3.0:1787826226.073195:0:63688:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86042600/0xe5f736940357b40c lrc: 3/0,1 mode: PW/PW res: [0x200000404:0x1f5:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0ee90 expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.073202:0:63688:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86042600/0xe5f736940357b40c lrc: 2/0,0 mode: PW/PW res: [0x200000404:0x1f5:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0ee90 expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200008:3.0:1787826226.073258:0:63688:0:(file.c:6313:ll_getattr_dentry()) START file f200c.sanity-flr:[0x200000404:0x1f5:0x0](ffff907d93322a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787826226.073266:0:63688:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1f5:0x0](ffff907d93322a08),name=f200c.sanity-flr 00000002:00010000:3.0:1787826226.073285:0:63688:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f5:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826226.073294:0:63688:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826226.073378:0:63688:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787826226.073385:0:63688:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826226.073391:0:63688:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f5:0x0](ffff907d93322a08) 00000080:00010000:3.0:1787826226.073394:0:63688:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f5:0x0](ffff907d93322a08) for lock 0xe5f736940357b3db 00000080:00010000:3.0:1787826226.073398:0:63688:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b3db from it ffff9dd140e33cd8 00010000:00010000:3.0:1787826226.073401:0:63688:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826226.073408:0:63688:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1787826226.073491:0:63688:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f5:0x0] ignore/verify layout 0/1, layout version 1 restore needed 0 00000001:00010000:3.0:1787826226.073502:0:63688:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1f5:0x0] 00000008:00010000:3.0:1787826226.073519:0:63688:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826226.073523:0:63688:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d98a1cab0, osc lock ffff907d963b9c08, flags 0x400001002 00010000:00010000:3.0:1787826226.073538:0:63688:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826226.073554:0:63688:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:3.0:1787826226.073564:0:63688:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826226.073568:0:63688:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826226.073575:0:63688:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 1 00000020:00200000:3.0:1787826226.073584:0:63688:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1787826226.073591:0:63688:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f5:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:3.0:1787826226.073595:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da61e98b0 (ffff907d8662b1b8/0) at cl_lock_release():245 00000020:00010000:3.0:1787826226.073598:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da61e98b0 (ffff907d8662b1b8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826226.073601:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98a1cab0 (ffff907da5608368/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826226.073856:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da61e98b0 (ffff907d8662b1b8/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826226.073861:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98a1cab0 (ffff907d8662b1b8/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826226.073872:0:63688:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f5:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:3.0:1787826226.073885:0:63688:0:(file.c:6313:ll_getattr_dentry()) START file f200c.sanity-flr:[0x200000404:0x1f5:0x0](ffff907d93322a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787826226.073889:0:63688:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1f5:0x0](ffff907d93322a08),name=f200c.sanity-flr 00000002:00010000:3.0:1787826226.073900:0:63688:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f5:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826226.073907:0:63688:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826226.073920:0:63688:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1787826226.073926:0:63688:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826226.073930:0:63688:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f5:0x0](ffff907d93322a08) 00000080:00010000:3.0:1787826226.073932:0:63688:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f5:0x0](ffff907d93322a08) for lock 0xe5f736940357b3db 00000080:00010000:3.0:1787826226.073936:0:63688:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b3db from it ffff9dd140e33cd8 00010000:00010000:3.0:1787826226.073938:0:63688:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826226.073945:0:63688:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 1 00000080:00200000:3.0:1787826226.073956:0:63688:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f5:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1787826226.073960:0:63688:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1f5:0x0] 00000008:00010000:3.0:1787826226.073968:0:63688:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826226.073970:0:63688:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d98a1cab0, osc lock ffff907d963b9c08, flags 0x400001002 00010000:00010000:3.0:1787826226.073978:0:63688:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 1 00010000:00010000:3.0:1787826226.073987:0:63688:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 1 00000020:00200000:3.0:1787826226.073996:0:63688:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826226.073999:0:63688:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 1 00010000:00010000:3.0:1787826226.074010:0:63688:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 1 00000020:00200000:3.0:1787826226.074018:0:63688:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1787826226.074022:0:63688:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f5:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:3.0:1787826226.074025:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da61e98b0 (ffff907d8662b1b8/0) at cl_lock_release():245 00000020:00010000:3.0:1787826226.074027:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da61e98b0 (ffff907d8662b1b8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826226.074030:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98a1cab0 (ffff907da5608368/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826226.074032:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da61e98b0 (ffff907d8662b1b8/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826226.074034:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98a1cab0 (ffff907d8662b1b8/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826226.074040:0:63688:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f5:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:3.0:1787826226.074049:0:63688:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1f5:0x0](ffff907d93322a08) cmd=801066da arg=7fffcbff71e0 00000080:00200000:3.0:1787826226.074056:0:63688:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f5:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:3.0:1787826226.074062:0:63688:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:2.0:1787826226.074174:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:d203faaf-7906-4d13-b1ba-8437467d6639:2350:1874670011354880:192.168.203.104@tcp:1:lfs.0 00000100:00100000:3.0:1787826226.076525:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:d203faaf-7906-4d13-b1ba-8437467d6639:2350:1874670011354880:192.168.203.104@tcp:1:lfs.0 00000080:00200000:2.0:1787826226.076735:0:63688:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1f5:0x0](ffff907d93322a08), inode mode 81a4 mask 4 00000080:00200000:2.0:1787826226.076746:0:63688:0:(xattr.c:760:ll_xattr_get()) VFS Op:inode=[0x200000404:0x1f5:0x0](ffff907d93322a08), xattr lov 00000080:00200008:2.0:1787826226.076775:0:63688:0:(file.c:6313:ll_getattr_dentry()) START file f200c.sanity-flr:[0x200000404:0x1f5:0x0](ffff907d93322a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1787826226.076783:0:63688:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1f5:0x0](ffff907d93322a08),name=f200c.sanity-flr 00000002:00010000:2.0:1787826226.076818:0:63688:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f5:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826226.076837:0:63688:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 1 00010000:00010000:2.0:1787826226.076851:0:63688:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 1 00000080:00200000:2.0:1787826226.076858:0:63688:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826226.076866:0:63688:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f5:0x0](ffff907d93322a08) 00000080:00010000:2.0:1787826226.076870:0:63688:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f5:0x0](ffff907d93322a08) for lock 0xe5f736940357b3db 00000080:00010000:2.0:1787826226.076877:0:63688:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b3db from it ffff9dd140e33cd8 00010000:00010000:2.0:1787826226.076882:0:63688:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 1 00010000:00010000:2.0:1787826226.076890:0:63688:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d86043e00/0xe5f736940357b3db lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0ed9b expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 1 00000080:00200000:2.0:1787826226.076912:0:63688:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f5:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1787826226.076923:0:63688:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1f5:0x0] 00000008:00010000:2.0:1787826226.076940:0:63688:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826226.076944:0:63688:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d963225b0, osc lock ffff907d830a2d68, flags 0x400001002 00010000:00010000:2.0:1787826226.076958:0:63688:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 1 00010000:00010000:2.0:1787826226.076969:0:63688:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 1 00000020:00200000:2.0:1787826226.076979:0:63688:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826226.076982:0:63688:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 1 00010000:00010000:2.0:1787826226.076989:0:63688:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00000020:00200000:2.0:1787826226.077001:0:63688:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826226.077009:0:63688:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f5:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826226.077013:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da61e98b0 (ffff907d8662b1b8/0) at cl_lock_release():245 00000020:00010000:2.0:1787826226.077016:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da61e98b0 (ffff907d8662b1b8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826226.077019:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d963225b0 (ffff907da5608368/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826226.077023:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da61e98b0 (ffff907d8662b1b8/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826226.077025:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d963225b0 (ffff907d8662b1b8/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826226.077034:0:63688:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f5:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:3.0:1787826226.236971:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826226.237077:0:40338:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:2.0:1787826226.237096:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025467392 00000100:00100000:2.0:1787826226.237104:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025467392 00000100:00100000:2.0:1787826226.237155:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7089 00000100:00100000:2.0:1787826226.237165:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83facc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:23593:x1874670025467392:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826226.237184:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d850a3000 lock: 0000000088389957/0xe5f736940357b3fe lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0ee82 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826226.237256:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826226.237264:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83facc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:23593:x1874670025467392:12345-192.168.203.104@tcp:104: Request processed in 100us (301us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826226.237271:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7089 00000100:00100000:3.0:1787826226.375934:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826226.376038:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025467520 00000100:00100000:2.0:1787826226.376048:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025467520 00000100:00100000:2.0:1787826226.376138:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7090 00000100:00100000:2.0:1787826226.376148:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83525680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:6932:x1874670025467520:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826226.376165:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d850a3000 lock: 0000000029c7e506/0xe5f736940357b405 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x303:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0ee7b expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826226.376247:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826226.376253:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83525680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:6932:x1874670025467520:12345-192.168.203.104@tcp:104: Request processed in 107us (330us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826226.376260:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7090 00000100:00100000:3.0:1787826226.384785:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826226.384978:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025467648 00000100:00100000:2.0:1787826226.384988:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025467648 00000100:00100000:2.0:1787826226.385041:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7091 00000100:00100000:2.0:1787826226.385050:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d835261c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:8014:x1874670025467648:12345-192.168.203.104@tcp:105: 00010000:00010000:2.0:1787826226.385067:0:50445:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff907d850a3000 lock: 000000008408fc3d/0xe5f736940357b428 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0eedd expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.385081:0:50445:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff907d850a3000 lock: 000000008408fc3d/0xe5f736940357b428 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0eedd expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.385090:0:50445:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff907d850a3000 lock: 000000008408fc3d/0xe5f736940357b428 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0eedd expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.385102:0:50445:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff907d850a3000 lock: 000000008408fc3d/0xe5f736940357b428 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10080000000000 nid: local remote: 0x7918213f4fc0eedd expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.385112:0:50445:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d850a3000 lock: 000000008408fc3d/0xe5f736940357b428 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10080000000000 nid: local remote: 0x7918213f4fc0eedd expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.385121:0:50445:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff907d850a3000 lock: 000000008408fc3d/0xe5f736940357b428 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80000000000 nid: local remote: 0x7918213f4fc0eedd expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.385139:0:50445:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff907d83f0b000) 00000100:00100000:2.0:1787826226.385198:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d835261c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:8014:x1874670025467648:12345-192.168.203.104@tcp:105: Request processed in 149us (423us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826226.385206:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7091 00000020:00010000:3.0:1787826226.394713:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da03eee48 (ffff907d83f04488/0) at cl_lock_release():245 00000020:00010000:3.0:1787826226.394719:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da03eee48 (ffff907d83f04488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826226.394723:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d927fcb70 (ffff907d8662ba28/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787826226.394735:0:63688:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d850a3000 lock: 0000000095f08f3d/0xe5f736940357b421 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0eed6 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.394757:0:63688:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d850a3000 lock: 0000000095f08f3d/0xe5f736940357b421 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0eed6 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:3.0:1787826226.394772:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d927fcbb0 (ffff907da56081b8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787826226.394777:0:63688:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d850a3000 lock: 000000008408fc3d/0xe5f736940357b428 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc0eedd expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826226.394786:0:63688:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d850a3000 lock: 000000008408fc3d/0xe5f736940357b428 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0eedd expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:3.0:1787826226.394811:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da03eee48 (ffff907d83f04488/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826226.394815:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d927fcb70 (ffff907d83f04488/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826226.394820:0:63688:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d927fcbb0 (ffff907d83f04488/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826226.394838:0:63688:0:(vvp_io.c:304:vvp_io_fini()) [0x200000bd1:0x1:0x0] ignore/verify layout 0/0, layout version 1 need write layout 0, restore needed 0, invalidate_lock 0 00800000:00000080:3.0:1787826226.394852:0:63688:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff907d850a3000: cmd=401866d4 len=392 karg=ffff907da574aa00 uarg=0000000000000000 00000002:00000080:3.0:1787826226.394861:0:63688:0:(mdc_request.c:2317:mdc_iocontrol()) lustre-MDT0000-mdc-ffff907d850a3000: cmd=401866d4 len=392 karg=ffff907da574aa00 uarg=0000000000000000 00000020:00100000:3.0:1787826226.394886:0:63688:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d850a3000: modify RPC slot 1 is allocated opc 55, max 7 00000100:00100000:3.0:1787826226.394894:0:63688:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83524f00 x1874670011357312/t0(0) o55->lustre-MDT0000-mdc-ffff907d850a3000@192.168.203.104@tcp:12/10 lens 472/224 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1787826226.394917:0:63688:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job lfs:d203faaf-7906-4d13-b1ba-8437467d6639:63688:1874670011357312:192.168.203.104@tcp:55:lfs.0 00000100:00100000:3.0:1787826226.395388:0:63688:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d98acd0c0 going to sleep for 11 seconds 00000100:00100000:3.0:1787826226.404915:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826226.405065:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025467776 00000100:00100000:2.0:1787826226.405076:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025467776 00000100:00100000:2.0:1787826226.405130:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7092 00000100:00100000:2.0:1787826226.405140:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d835270c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:d203faaf-7906-4d13-b1ba-8437467d6639+5:36493:x1874670025467776:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826226.405157:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a200/0xe5f736940357b413 lrc: 2/0,0 mode: PR/PR res: [0x200000bd1:0x1:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x7918213f4fc0eeb3 expref: -99 pid: 63688 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826226.405235:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826226.405241:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d835270c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:d203faaf-7906-4d13-b1ba-8437467d6639+5:36493:x1874670025467776:12345-192.168.203.104@tcp:104: Request processed in 103us (335us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826226.405247:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7092 00000100:00100000:2.0:1787826226.440099:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826226.440183:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025467904 00000100:00100000:3.0:1787826226.440193:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025467904 00000100:00100000:3.0:1787826226.440245:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7093 00000100:00100000:3.0:1787826226.440254:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d9114bc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:36728:x1874670025467904:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826226.440272:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000f59132ed/0xe5f736940357b3f0 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x31d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0ee66 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:3.0:1787826226.440347:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826226.440353:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d9114bc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:36728:x1874670025467904:12345-192.168.203.104@tcp:104: Request processed in 101us (263us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826226.440360:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7093 00000100:00100000:3.0:1787826226.445186:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826226.447052:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025468032 00000100:00100000:2.0:1787826226.447065:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025468032 00000100:00100000:2.0:1787826226.447129:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7094 00000100:00100000:2.0:1787826226.447139:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83527840 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:28563:x1874670025468032:12345-192.168.203.104@tcp:105: 00010000:00010000:2.0:1787826226.447155:0:50445:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000c925444c/0xe5f736940357b436 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0eef9 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.447168:0:50445:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000c925444c/0xe5f736940357b436 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0eef9 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.447177:0:50445:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000c925444c/0xe5f736940357b436 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0eef9 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.447187:0:50445:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000c925444c/0xe5f736940357b436 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10080000000000 nid: local remote: 0x7918213f4fc0eef9 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.447200:0:50445:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000c925444c/0xe5f736940357b436 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10080000000000 nid: local remote: 0x7918213f4fc0eef9 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.447209:0:50445:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000c925444c/0xe5f736940357b436 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80000000000 nid: local remote: 0x7918213f4fc0eef9 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826226.447226:0:50445:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff907d8883a600) 00000100:00100000:2.0:1787826226.447285:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83527840 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:28563:x1874670025468032:12345-192.168.203.104@tcp:105: Request processed in 147us (2108us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826226.447292:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7094 00000100:00100000:3.0:1787826226.466072:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826226.466213:0:40339:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:2.0:1787826226.466232:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025468160 00000100:00100000:2.0:1787826226.466240:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025468160 00000100:00100000:2.0:1787826226.466295:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7095 00000100:00100000:2.0:1787826226.466304:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d8601c450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:d203faaf-7906-4d13-b1ba-8437467d6639+5:32459:x1874670025468160:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826226.466321:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883a200/0xe5f736940357b42f lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0eef2 expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826226.466402:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826226.466409:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d8601c450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:d203faaf-7906-4d13-b1ba-8437467d6639+5:32459:x1874670025468160:12345-192.168.203.104@tcp:104: Request processed in 106us (347us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826226.466417:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7095 00000100:00100000:3.0:1787826226.476985:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826226.477069:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025468288 00000100:00100000:2.0:1787826226.477079:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025468288 00000100:00100000:2.0:1787826226.477135:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7096 00000100:00100000:2.0:1787826226.477146:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83524000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:d203faaf-7906-4d13-b1ba-8437467d6639+5:32459:x1874670025468288:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826226.477162:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d83f0b200/0xe5f736940357b41a lrc: 2/0,0 mode: CR/CR res: [0x200000bd1:0x1:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0eecf expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826226.477279:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826226.477287:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83524000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:d203faaf-7906-4d13-b1ba-8437467d6639+5:32459:x1874670025468288:12345-192.168.203.104@tcp:104: Request processed in 143us (311us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826226.477294:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7096 00010000:00010000:2.0:1787826227.064043:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826229.251241:0:63559:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b3cd lrc: 4/0,1 mode: --/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef1c expref: -99 pid: 63559 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.251253:0:63559:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b3cd lrc: 4/0,1 mode: --/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef1c expref: -99 pid: 63559 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.251264:0:63559:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b3cd lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef1c expref: -99 pid: 63559 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.251274:0:63559:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b3cd lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef1c expref: -99 pid: 63559 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.251288:0:63559:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b3cd lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef1c expref: -99 pid: 63559 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.251296:0:63559:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b3cd lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef1c expref: -99 pid: 63559 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.251303:0:63559:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b3cd lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef1c expref: -99 pid: 63559 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826229.251812:0:63559:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x2b, status=0 req@ffff907d90845a40 x1874670011351808/t8589942879(8589942879) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/872 e 0 to 0 dl 1787826239 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1787826229.251825:0:63559:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90845a40 x1874670011351808/t8589942879(8589942879) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/872 e 0 to 0 dl 1787826239 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1787826229.251837:0:63559:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 264 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b3cd lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef1c expref: -99 pid: 63559 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1787826229.251851:0:63559:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b3cd lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef1c expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.251863:0:63559:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/501 (0 0) 00000080:00200000:2.0:1787826229.251869:0:63559:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1787826229.251912:0:63559:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f5:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1787826229.251922:0:63559:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1787826229.251948:0:63559:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f5:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826229.252010:0:63559:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x1f5:0x0](ffff907d9221e3c8): rc = 0 00000080:00200000:2.0:1787826229.252028:0:63559:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1f5:0x0]: layout version change: 4294967294 -> 1 00000080:00010000:2.0:1787826229.252034:0:63559:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f5:0x0](ffff907d9221e3c8) for lock 0xe5f736940357b3cd 00000080:00010000:2.0:1787826229.252044:0:63559:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f5:0x0](ffff907d9221e3c8) 00000080:00200008:2.0:1787826229.252060:0:63559:0:(file.c:1090:ll_file_open()) START file f200c.sanity-flr:[0x200000404:0x1f5:0x0](ffff907d9221e3c8), flags 140101 00000002:00100000:2.0:1787826229.252079:0:63559:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90845a40 x1874670011351808/t8589942879(8589942879) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 800/872 e 0 to 0 dl 1787826239 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1787826229.252118:0:63559:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b3cd from it ffff907d83241360 00010000:00010000:2.0:1787826229.252122:0:63559:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b3cd lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0ef1c expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.252133:0:63559:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b3cd lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0ef1c expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826229.252156:0:63559:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1f5:0x0](ffff907d9221e3c8), inode mode 81a4 mask 42 00000080:00200008:3.0:1787826229.298979:0:63559:0:(file.c:2652:do_file_write_iter()) START file f200c.sanity-flr:[0x200000404:0x1f5:0x0], ppos: 0, count: 1048576 00000080:00200000:3.0:1787826229.298996:0:63559:0:(file.c:2090:ll_file_io_generic()) f200c.sanity-flr: write ppos: 0, bytes: 1048576 00000080:00200000:3.0:1787826229.299009:0:63559:0:(file.c:1848:ll_io_set_mirror()) f200c.sanity-flr: desiginated mirror: 0 00000020:00200000:3.0:1787826229.299018:0:63559:0:(cl_io.c:218:cl_io_rw_init()) header@000000000d663f42[0x110, 2, [0x200000404:0x1f5:0x0] hash] 00000020:00200000:3.0:1787826229.299021:0:63559:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [0, 1048576) 0 0 00000080:00200000:3.0:1787826229.299027:0:63559:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f5:0x0] ignore/verify layout 0/0, layout version 1 restore needed 0 00020000:00200000:3.0:1787826229.299042:0:63559:0:(lov_io.c:571:lov_io_mirror_init()) [0x200000404:0x1f5:0x0] need write intent for [0, 1048576) 00000020:00200000:3.0:1787826229.299047:0:63559:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1787826229.299053:0:63559:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f5:0x0] ignore/verify layout 0/0, layout version 1 need write layout 1, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787826229.299056:0:63559:0:(vvp_io.c:371:vvp_io_fini()) [0x200000404:0x1f5:0x0] write layout, type 2 [0x0, 0x100000) 00000080:00010000:3.0:1787826229.299067:0:63559:0:(file.c:7076:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000404:0x1f5:0x0](ffff907d9221e3c8) 00000002:00010000:3.0:1787826229.299081:0:63559:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f5:0x0]) in obj [0x200000404:0x1f5:0x0], intent: layout flags 02 00010000:00010000:3.0:1787826229.299105:0:63559:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b3cd lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0ef1c expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787826229.299131:0:63559:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x1f5:0x0]: losing layout lock 00010000:00010000:3.0:1787826229.299173:0:63559:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b3cd lrc: 1/0,0 mode: --/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0ef1c expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826229.299180:0:63559:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787826229.299183:0:63559:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b3cd lrc: 0/0,0 mode: --/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0ef1c expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826229.299220:0:63559:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 3/0,1 mode: --/EX res: [0x200000404:0x1f5:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 63559 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826229.299227:0:63559:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 3/0,1 mode: --/EX res: [0x200000404:0x1f5:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63559 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826229.299237:0:63559:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826229.299248:0:63559:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83524780 x1874670011359616/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/2048 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826229.299258:0:63559:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 3/0,1 mode: --/EX res: [0x200000404:0x1f5:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63559 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826229.299278:0:63559:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:63559:1874670011359616:192.168.203.104@tcp:101:dd.0 00000100:00100000:3.0:1787826229.299384:0:63559:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d98acd240 going to sleep for 13 seconds 00000100:00100000:3.0:1787826229.303323:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826229.304025:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025469312 00000100:00100000:2.0:1787826229.304038:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025469312 00000100:00100000:2.0:1787826229.304099:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7097 00000100:00100000:2.0:1787826229.304110:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d835252c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:d203faaf-7906-4d13-b1ba-8437467d6639+5:32457:x1874670025469312:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826229.304128:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d8883ae00/0xe5f736940357b43d lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0ef0e expref: -99 pid: 63688 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826229.304220:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826229.304227:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d835252c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:d203faaf-7906-4d13-b1ba-8437467d6639+5:32457:x1874670025469312:12345-192.168.203.104@tcp:104: Request processed in 119us (916us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826229.304235:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7097 00000100:00100000:2.0:1787826229.318021:0:63559:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589942880 gen 1 00000100:00100000:2.0:1787826229.318041:0:63559:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d90845a40 x1874670011351808/t8589942879(8589942879) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 800/872 e 0 to 0 dl 1787826239 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1787826229.318053:0:63559:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:63559:1874670011359616:192.168.203.104@tcp:101:dd.0 00010000:00010000:2.0:1787826229.318075:0:63559:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 4/0,1 mode: --/EX res: [0x200000404:0x1f5:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.318087:0:63559:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 4/0,1 mode: --/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.318103:0:63559:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 4/0,1 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.318113:0:63559:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 4/0,1 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.318125:0:63559:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 5/1,1 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.318133:0:63559:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 5/1,1 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.318140:0:63559:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826229.318149:0:63559:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x400 disposition=0x1, status=0 req@ffff907d83524780 x1874670011359616/t8589942880(8589942880) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/608 e 0 to 0 dl 1787826247 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1787826229.318171:0:63559:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 264 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1787826229.318182:0:63559:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.318193:0:63559:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/501 (0 0) 00000080:00200000:2.0:1787826229.318199:0:63559:0:(namei.c:952:ll_intent_lock()) intent lock 1024 on i1 [0x200000404:0x1f5:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1787826229.318215:0:63559:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f5:0x0](ffff907d9221e3c8) for lock 0xe5f736940357b444 00000080:00010000:2.0:1787826229.318225:0:63559:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b444 from it ffff9dd14203fbb8 00010000:00010000:2.0:1787826229.318229:0:63559:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.318237:0:63559:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826229.318254:0:63559:0:(file.c:2240:ll_file_io_generic()) f200c.sanity-flr: 2 io complete with rc: 0, result: 0, restart: 1 00000080:00200000:2.0:1787826229.318257:0:63559:0:(file.c:2247:ll_file_io_generic()) f200c.sanity-flr: restart write from ppos=0 bytes=1048576 retries=999 ret=0: rc = 0 00000080:00200000:2.0:1787826229.318264:0:63559:0:(file.c:1848:ll_io_set_mirror()) f200c.sanity-flr: desiginated mirror: 0 00000020:00200000:2.0:1787826229.318272:0:63559:0:(cl_io.c:218:cl_io_rw_init()) header@000000000d663f42[0x110, 2, [0x200000404:0x1f5:0x0] hash] 00000020:00200000:2.0:1787826229.318274:0:63559:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [0, 1048576) 0 0 00000080:00200000:2.0:1787826229.318279:0:63559:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f5:0x0] ignore/verify layout 0/0, layout version 1 restore needed 0 00010000:00010000:2.0:1787826229.318295:0:63559:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826229.318305:0:63559:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1787826229.318312:0:63559:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x1f5:0x0](00000000a3aeb7eb) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826229.318350:0:63559:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826229.318358:0:63559:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1787826229.318372:0:63559:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f5:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1787826229.318381:0:63559:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1f5:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826229.318384:0:63559:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1787826229.318390:0:63559:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1787826229.318413:0:63559:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f5:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1787826229.318425:0:63559:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826229.318433:0:63559:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1787826229.318441:0:63559:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x1f5:0x0](00000000a3aeb7eb) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1787826229.318457:0:63559:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f5:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1787826229.318461:0:63559:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1f5:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826229.318464:0:63559:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1787826229.318466:0:63559:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1787826229.318475:0:63559:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f5:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826229.318540:0:63559:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1f5:0x0]: layout version change: 4294967294 -> 2 00010000:00010000:2.0:1787826229.318545:0:63559:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826229.318552:0:63559:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042400/0xe5f736940357b444 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f5:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0ef31 expref: -99 pid: 63559 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00020000:00200000:2.0:1787826229.318562:0:63559:0:(lov_io.c:311:lov_io_mirror_write_intent()) [0x200000404:0x1f5:0x0]there are 0 components to be staled to modify file extent [0x0, 0xffffffffffffffff), iot: 2 00020000:00200000:2.0:1787826229.318577:0:63559:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1f5:0x0]: flr state: 2, move mirror from 1 to 1, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826229.318579:0:63559:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00000080:00200000:2.0:1787826229.318584:0:63559:0:(file.c:2170:ll_file_io_generic()) Range lock [0, 255] 00020000:00200000:2.0:1787826229.318592:0:63559:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:2.0:1787826229.318594:0:63559:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00000080:00200000:2.0:1787826229.318652:0:63559:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 255] 00000008:00010000:2.0:1787826229.318665:0:63559:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1787826229.318670:0:63559:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d963225b0, osc lock ffff907d830a2d68, flags 0x0 00000080:00200000:2.0:1787826229.318678:0:63559:0:(vvp_io.c:1339:vvp_io_write_start()) f200c.sanity-flr: write [0, 1048576) append 0 append lockpos 0 00000002:00010000:2.0:1787826229.318704:0:63559:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x0:0x0:0x0]) in obj [0x200000404:0x1f5:0x0], intent: getxattr flags 0 00010000:00010000:2.0:1787826229.318748:0:63559:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18400/0xe5f736940357b44b lrc: 3/1,0 mode: --/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 63559 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.318755:0:63559:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18400/0xe5f736940357b44b lrc: 3/1,0 mode: --/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63559 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.318764:0:63559:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18400/0xe5f736940357b44b lrc: 3/1,0 mode: --/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63559 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826229.318793:0:63559:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:63559:1874670011359872:192.168.203.104@tcp:101:dd.0 00000100:00100000:2.0:1787826229.318879:0:63559:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c02240 going to sleep for 13 seconds 00000100:00100000:2.0:1787826229.331367:0:63559:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:63559:1874670011359872:192.168.203.104@tcp:101:dd.0 00010000:00010000:2.0:1787826229.331401:0:63559:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18400/0xe5f736940357b44b lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef3f expref: -99 pid: 63559 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.331418:0:63559:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18400/0xe5f736940357b44b lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef3f expref: -99 pid: 63559 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.331428:0:63559:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18400/0xe5f736940357b44b lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef3f expref: -99 pid: 63559 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826229.331440:0:63559:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x80 disposition=0x0, status=0 req@ffff907d9114bc00 x1874670011359872/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 568/1360 e 0 to 0 dl 1787826247 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1787826229.331452:0:63559:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18400/0xe5f736940357b44b lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef3f expref: -99 pid: 63559 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.331465:0:63559:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/501 (0 0) 00000080:00200000:2.0:1787826229.331471:0:63559:0:(namei.c:952:ll_intent_lock()) intent lock 128 on i1 [0x200000404:0x1f5:0x0] suppgids 0 -1: rc 0 00000080:00010000:2.0:1787826229.331493:0:63559:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f5:0x0](ffff907d9221e3c8) for lock 0xe5f736940357b44b 00000080:00010000:2.0:1787826229.331501:0:63559:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b44b from it ffff9dd14203f8e8 00010000:00010000:2.0:1787826229.331505:0:63559:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18400/0xe5f736940357b44b lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef3f expref: -99 pid: 63559 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826229.331513:0:63559:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18400/0xe5f736940357b44b lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef3f expref: -99 pid: 63559 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826229.331550:0:63559:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x1f5:0x0](ffff907d9221e3c8), size=1048576 (max 335544320), offset=0=0x0, pages 256 (max 81920), lockless, parallel 00000020:00200000:2.0:1787826229.331922:0:63559:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1787826229.331926:0:63559:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 1048576 00000020:00010000:2.0:1787826229.331936:0:63559:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da03eee48 (ffff907d83f04488/0) at cl_lock_release():245 00000020:00010000:2.0:1787826229.331940:0:63559:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da03eee48 (ffff907d83f04488/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826229.331944:0:63559:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d963225b0 (ffff907d8662b1b8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826229.331947:0:63559:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da03eee48 (ffff907d83f04488/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826229.331950:0:63559:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d963225b0 (ffff907d83f04488/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826229.331958:0:63559:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1787826229.331964:0:63559:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1787826229.331983:0:61492:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907da56f8e40 00000008:00100000:2.0:1787826229.332492:0:61492:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d9114bc00 - object 0x240000400:798 offset 0<>1048576 00000100:00100000:3.0:1787826229.333403:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011360000:192.168.203.104@tcp:4:dd.0 00000100:00100000:3.0:1787826229.341961:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826229.342329:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025469824 00000100:00100000:2.0:1787826229.342339:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025469824 00000100:00100000:2.0:1787826229.342406:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7098 00000100:00100000:2.0:1787826229.342415:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83526580 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:23594:x1874670025469824:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826229.342433:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d850a3000 lock: 0000000095f08f3d/0xe5f736940357b421 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0eed6 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826229.342517:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826229.342524:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83526580 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:23594:x1874670025469824:12345-192.168.203.104@tcp:104: Request processed in 110us (574us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826229.342531:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7098 00000100:00100000:2.0:1787826229.406806:0:2351:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff907d986d0000: committing for last_committed 21474840474 gen 3 00000008:00010000:2.0:1787826229.406877:0:2351:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907da56f8e40 00000020:00200000:2.0:1787826229.406885:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1787826229.406888:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1787826229.406890:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1787826229.406892:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1787826229.406894:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1787826229.406895:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1787826229.406897:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1787826229.406911:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1787826229.406912:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1787826229.406914:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1787826229.406915:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1787826229.406917:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1787826229.406918:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1787826229.406920:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1787826229.406921:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1787826229.406923:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1787826229.406925:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1787826229.406926:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1787826229.406928:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1787826229.406930:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1787826229.406932:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1787826229.406934:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1787826229.406936:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1787826229.406937:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1787826229.406939:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1787826229.406940:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1787826229.406942:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1787826229.406944:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1787826229.406945:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1787826229.406947:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1787826229.406948:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1787826229.406950:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1787826229.406952:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1787826229.406953:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1787826229.406962:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1787826229.406964:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1787826229.406966:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1787826229.406968:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1787826229.406970:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1787826229.406971:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1787826229.406973:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1787826229.406975:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1787826229.406977:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1787826229.406978:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1787826229.406980:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1787826229.406982:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1787826229.406983:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1787826229.406985:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1787826229.406986:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1787826229.406988:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1787826229.406989:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1787826229.406991:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1787826229.406993:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1787826229.406994:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1787826229.406996:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1787826229.406998:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1787826229.406999:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1787826229.407000:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1787826229.407001:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1787826229.407003:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1787826229.407004:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1787826229.407005:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1787826229.407007:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1787826229.407008:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1787826229.407010:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1787826229.407011:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1787826229.407013:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1787826229.407014:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1787826229.407016:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1787826229.407017:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1787826229.407018:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1787826229.407020:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1787826229.407021:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1787826229.407023:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1787826229.407024:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1787826229.407026:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1787826229.407027:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1787826229.407028:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1787826229.407030:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1787826229.407031:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1787826229.407033:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1787826229.407034:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1787826229.407036:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1787826229.407038:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1787826229.407040:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1787826229.407041:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1787826229.407043:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1787826229.407044:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1787826229.407046:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1787826229.407047:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1787826229.407049:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1787826229.407050:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1787826229.407051:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1787826229.407053:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1787826229.407054:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1787826229.407056:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1787826229.407057:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1787826229.407059:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1787826229.407060:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1787826229.407062:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1787826229.407064:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1787826229.407066:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1787826229.407067:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1787826229.407069:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1787826229.407071:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1787826229.407072:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1787826229.407074:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1787826229.407076:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1787826229.407077:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1787826229.407079:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1787826229.407080:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1787826229.407081:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1787826229.407082:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1787826229.407084:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1787826229.407085:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1787826229.407087:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1787826229.407089:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1787826229.407090:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1787826229.407092:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1787826229.407093:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1787826229.407095:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1787826229.407096:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1787826229.407098:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1787826229.407099:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1787826229.407101:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1787826229.407102:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1787826229.407104:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1787826229.407105:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1787826229.407106:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1787826229.407108:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1787826229.407110:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1787826229.407112:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1787826229.407113:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1787826229.407115:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1787826229.407117:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1787826229.407119:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1787826229.407120:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1787826229.407122:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1787826229.407124:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1787826229.407125:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1787826229.407127:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1787826229.407129:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1787826229.407130:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1787826229.407133:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1787826229.407135:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1787826229.407136:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1787826229.407138:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1787826229.407140:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1787826229.407141:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1787826229.407143:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1787826229.407144:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1787826229.407146:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1787826229.407147:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1787826229.407149:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1787826229.407151:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1787826229.407152:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1787826229.407154:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1787826229.407155:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1787826229.407156:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1787826229.407158:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1787826229.407159:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1787826229.407161:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1787826229.407163:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1787826229.407165:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1787826229.407166:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1787826229.407168:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1787826229.407170:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1787826229.407171:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1787826229.407173:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1787826229.407174:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1787826229.407176:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1787826229.407177:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1787826229.407179:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1787826229.407180:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1787826229.407182:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1787826229.407184:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1787826229.407186:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1787826229.407187:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1787826229.407189:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1787826229.407191:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1787826229.407192:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1787826229.407194:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1787826229.407195:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1787826229.407196:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1787826229.407197:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1787826229.407199:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1787826229.407200:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1787826229.407202:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1787826229.407204:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1787826229.407205:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1787826229.407206:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1787826229.407208:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1787826229.407209:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1787826229.407211:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1787826229.407213:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1787826229.407215:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1787826229.407216:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1787826229.407218:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1787826229.407219:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1787826229.407221:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1787826229.407222:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1787826229.407224:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1787826229.407225:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1787826229.407227:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1787826229.407229:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1787826229.407230:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1787826229.407232:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1787826229.407234:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1787826229.407235:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1787826229.407237:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1787826229.407239:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1787826229.407241:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1787826229.407243:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1787826229.407244:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1787826229.407245:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1787826229.407247:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1787826229.407249:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1787826229.407250:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1787826229.407252:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1787826229.407253:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1787826229.407255:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1787826229.407257:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1787826229.407258:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1787826229.407260:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1787826229.407261:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1787826229.407263:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1787826229.407264:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1787826229.407266:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1787826229.407268:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1787826229.407269:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1787826229.407271:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1787826229.407272:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1787826229.407274:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1787826229.407275:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1787826229.407277:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1787826229.407278:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1787826229.407279:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1787826229.407281:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1787826229.407282:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1787826229.407284:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1787826229.407285:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1787826229.407287:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1787826229.407288:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1787826229.407290:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1787826229.407292:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1787826229.407293:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1787826229.407295:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1787826229.407296:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1787826229.407297:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1787826229.407299:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1787826229.407300:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1787826229.407302:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1787826229.407303:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1787826229.407304:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1787826229.407306:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1787826229.407308:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1787826229.407401:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1787826229.407425:0:2351:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907da56f8e40 00000100:00100000:2.0:1787826229.407451:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011360000:192.168.203.104@tcp:4:dd.0 00000080:00200000:3.0:1787826229.407682:0:63559:0:(file.c:2207:ll_file_io_generic()) Range unlock [0, 255] 00000080:00200000:3.0:1787826229.407694:0:63559:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f5:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787826229.407700:0:63559:0:(file.c:2240:ll_file_io_generic()) f200c.sanity-flr: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:3.0:1787826229.407712:0:63559:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f5:0x0] updating i_size 1048576 i_blocks 4096 00000020:00200000:3.0:1787826229.407715:0:63559:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1787826229.407723:0:63559:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:2.0:1787826229.430848:0:63559:0:(file.c:2652:do_file_write_iter()) START file f200c.sanity-flr:[0x200000404:0x1f5:0x0], ppos: 1048576, count: 1048576 00000080:00200000:2.0:1787826229.430868:0:63559:0:(file.c:2090:ll_file_io_generic()) f200c.sanity-flr: write ppos: 1048576, bytes: 1048576 00000080:00200000:2.0:1787826229.430881:0:63559:0:(file.c:1848:ll_io_set_mirror()) f200c.sanity-flr: desiginated mirror: 0 00000020:00200000:2.0:1787826229.430890:0:63559:0:(cl_io.c:218:cl_io_rw_init()) header@000000000d663f42[0x110, 2, [0x200000404:0x1f5:0x0] hash] 00000020:00200000:2.0:1787826229.430893:0:63559:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1048576, 2097152) 0 0 00000080:00200000:2.0:1787826229.430899:0:63559:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f5:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1787826229.432193:0:63559:0:(lov_io.c:311:lov_io_mirror_write_intent()) [0x200000404:0x1f5:0x0]there are 0 components to be staled to modify file extent [0x0, 0xffffffffffffffff), iot: 2 00020000:00200000:2.0:1787826229.432201:0:63559:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1f5:0x0]: flr state: 2, move mirror from 1 to 1, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826229.432204:0:63559:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00000080:00200000:2.0:1787826229.432213:0:63559:0:(file.c:2170:ll_file_io_generic()) Range lock [256, 511] 00020000:00200000:2.0:1787826229.432221:0:63559:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 0 chunk: [1048576, 2097152) 2097152, 1048576 00020000:00200000:2.0:1787826229.432224:0:63559:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00000080:00200000:2.0:1787826229.432245:0:63559:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [256, 511] 00000008:00010000:2.0:1787826229.432264:0:63559:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1787826229.432269:0:63559:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d963225b0, osc lock ffff907d830a2d68, flags 0x0 00000080:00200000:2.0:1787826229.432280:0:63559:0:(vvp_io.c:1339:vvp_io_write_start()) f200c.sanity-flr: write [1048576, 2097152) append 0 append lockpos 0 00000080:00200000:2.0:1787826229.432291:0:63559:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x1f5:0x0](ffff907d9221e3c8), size=1048576 (max 335544320), offset=1048576=0x100000, pages 256 (max 81920), lockless, parallel 00000020:00200000:2.0:1787826229.432869:0:63559:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1787826229.432874:0:63559:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 2097152 00000020:00010000:2.0:1787826229.432884:0:63559:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da70e7038 (ffff907da5608368/0) at cl_lock_release():245 00000020:00010000:2.0:1787826229.432887:0:63559:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da70e7038 (ffff907da5608368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826229.432890:0:63559:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d963225b0 (ffff907d83f04878/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826229.432894:0:63559:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da70e7038 (ffff907da5608368/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826229.432896:0:63559:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d963225b0 (ffff907da5608368/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826229.433275:0:63559:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1787826229.433283:0:63559:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1787826229.433303:0:61492:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907da56f8e40 00000008:00100000:2.0:1787826229.434268:0:61492:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d9114bc00 - object 0x240000400:798 offset 1048576<>2097152 00000100:00100000:3.0:1787826229.434339:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011360256:192.168.203.104@tcp:4:dd.0 00000100:00100000:2.0:1787826229.502603:0:2350:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff907d986d0000: committing for last_committed 21474840475 gen 3 00000008:00010000:2.0:1787826229.502712:0:2350:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907da56f8e40 00000020:00200000:2.0:1787826229.502721:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1787826229.502723:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1787826229.502725:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1787826229.502726:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1787826229.502728:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1787826229.502730:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1787826229.502732:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1787826229.502733:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1787826229.502735:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1787826229.502736:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1787826229.502738:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1787826229.502740:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1787826229.502741:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1787826229.502744:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1787826229.502745:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1787826229.502747:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1787826229.502749:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1787826229.502750:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1787826229.502752:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1787826229.502753:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1787826229.502754:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1787826229.502756:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1787826229.502757:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1787826229.502759:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1787826229.502760:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1787826229.502762:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1787826229.502763:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1787826229.502765:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1787826229.502767:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1787826229.502768:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1787826229.502770:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1787826229.502772:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1787826229.502783:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1787826229.502785:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1787826229.502787:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1787826229.502789:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1787826229.502790:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1787826229.502791:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1787826229.502793:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1787826229.502794:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1787826229.502796:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1787826229.502797:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1787826229.502799:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1787826229.502800:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1787826229.502801:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1787826229.502803:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1787826229.502804:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1787826229.502806:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1787826229.502807:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1787826229.502809:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1787826229.502811:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1787826229.502812:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1787826229.502814:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1787826229.502816:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1787826229.502818:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1787826229.502819:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1787826229.502821:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1787826229.502822:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1787826229.502824:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1787826229.502826:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1787826229.502827:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1787826229.502829:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1787826229.502830:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1787826229.502832:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1787826229.502833:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1787826229.502835:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1787826229.502836:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1787826229.502838:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1787826229.502839:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1787826229.502841:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1787826229.502843:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1787826229.502845:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1787826229.502846:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1787826229.503321:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1787826229.503327:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1787826229.503329:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1787826229.503330:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1787826229.503332:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1787826229.503333:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1787826229.503336:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1787826229.503337:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1787826229.503339:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1787826229.503341:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1787826229.503343:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1787826229.503345:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1787826229.503346:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1787826229.503348:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1787826229.503349:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1787826229.503351:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1787826229.503353:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1787826229.503355:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1787826229.503356:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1787826229.503358:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1787826229.503359:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1787826229.503361:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1787826229.503363:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1787826229.503365:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1787826229.503366:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1787826229.503368:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1787826229.503370:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1787826229.503372:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1787826229.503373:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1787826229.503375:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1787826229.503377:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1787826229.503378:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1787826229.503380:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1787826229.503381:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1787826229.503383:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1787826229.503384:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1787826229.503385:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1787826229.503387:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1787826229.503389:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1787826229.503391:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1787826229.503392:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1787826229.503394:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1787826229.503395:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1787826229.503397:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1787826229.503398:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1787826229.503400:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1787826229.503401:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1787826229.503403:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1787826229.503404:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1787826229.503406:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1787826229.503408:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1787826229.503410:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1787826229.503412:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1787826229.503414:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1787826229.503416:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1787826229.503417:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1787826229.503419:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1787826229.503421:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1787826229.503422:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1787826229.503424:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1787826229.503426:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1787826229.503427:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1787826229.503429:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1787826229.503431:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1787826229.503432:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1787826229.503434:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1787826229.503435:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1787826229.503437:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1787826229.503438:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1787826229.503440:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1787826229.503442:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1787826229.503443:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1787826229.503445:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1787826229.503446:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1787826229.503448:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1787826229.503450:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1787826229.503451:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1787826229.503453:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1787826229.503455:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1787826229.503457:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1787826229.503458:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1787826229.503460:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1787826229.503461:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1787826229.503463:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1787826229.503465:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1787826229.503467:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1787826229.503469:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1787826229.503470:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1787826229.503472:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1787826229.503473:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1787826229.503475:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1787826229.503480:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1787826229.503482:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1787826229.503484:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1787826229.503486:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1787826229.503488:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1787826229.503489:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1787826229.503491:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1787826229.503492:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1787826229.503494:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1787826229.503495:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1787826229.503497:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1787826229.503499:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1787826229.503501:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1787826229.503502:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1787826229.503504:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1787826229.503505:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1787826229.503507:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1787826229.503508:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1787826229.503510:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1787826229.503511:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1787826229.503513:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1787826229.503514:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1787826229.503516:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1787826229.503518:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1787826229.503519:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1787826229.503521:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1787826229.503523:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1787826229.503525:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1787826229.503527:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1787826229.503528:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1787826229.503530:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1787826229.503531:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1787826229.503533:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1787826229.503534:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1787826229.503535:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1787826229.503537:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1787826229.503539:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1787826229.503540:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1787826229.503542:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1787826229.503543:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1787826229.503545:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1787826229.503547:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1787826229.503549:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1787826229.503551:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1787826229.503553:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1787826229.503555:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1787826229.503556:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1787826229.503558:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1787826229.503559:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1787826229.503711:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1787826229.503714:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1787826229.503716:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1787826229.503717:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1787826229.503719:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1787826229.503721:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1787826229.503722:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1787826229.503724:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1787826229.503726:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1787826229.503727:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1787826229.503729:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1787826229.503730:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1787826229.503732:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1787826229.503734:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1787826229.503735:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1787826229.503737:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1787826229.503739:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1787826229.503741:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1787826229.503742:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1787826229.503744:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1787826229.503745:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1787826229.503747:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1787826229.503749:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1787826229.503751:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1787826229.503752:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1787826229.503754:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1787826229.503756:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1787826229.503758:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1787826229.503759:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1787826229.503761:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1787826229.503763:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1787826229.503764:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1787826229.503766:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1787826229.503768:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1787826229.503769:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1787826229.503770:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1787826229.503772:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1787826229.503784:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1787826229.503786:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1787826229.503787:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1787826229.503789:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1787826229.503791:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1787826229.503793:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1787826229.503904:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1787826229.504035:0:2350:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907da56f8e40 00000100:00100000:2.0:1787826229.504064:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011360256:192.168.203.104@tcp:4:dd.0 00000080:00200000:3.0:1787826229.511380:0:63559:0:(file.c:2207:ll_file_io_generic()) Range unlock [256, 511] 00000080:00200000:3.0:1787826229.511403:0:63559:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f5:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787826229.511409:0:63559:0:(file.c:2240:ll_file_io_generic()) f200c.sanity-flr: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:3.0:1787826229.511424:0:63559:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f5:0x0] updating i_size 2097152 i_blocks 4096 00000020:00200000:3.0:1787826229.511429:0:63559:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1787826229.511444:0:63559:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:3.0:1787826229.512075:0:63559:0:(file.c:479:ll_file_release()) START file f200c.sanity-flr:[0x200000404:0x1f5:0x0](ffff907d9221e3c8), flags 140001 00000002:00100000:3.0:1787826229.513947:0:63559:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d90845a40 x1874670011351808/t8589942879(8589942879) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 800/872 e 0 to 0 dl 1787826239 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1787826229.513973:0:63559:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1787826229.513979:0:63559:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83526940 x1874670011360512/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1787826229.514006:0:63559:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83526940 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:63559:1874670011360512:192.168.203.104@tcp:35:dd.0 00000100:00100000:3.0:1787826229.514118:0:63559:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d98acda80 going to sleep for 11 seconds 00000100:00100000:2.0:1787826229.518174:0:63559:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83526940 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:63559:1874670011360512:192.168.203.104@tcp:35:dd.0 00000002:00100000:2.0:1787826229.518206:0:63559:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d90845a40 x1874670011351808/t8589942879(8589942879) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 800/872 e 0 to 0 dl 1787826239 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1787826231.032738:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011360640:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826231.032841:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011360896:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826231.032871:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:d203faaf-7906-4d13-b1ba-8437467d6639:2351:1874670011361152:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826231.032893:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:d203faaf-7906-4d13-b1ba-8437467d6639:2351:1874670011361408:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826231.032917:0:2351:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:2.0:1787826231.032923:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011360768:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826231.032945:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011361024:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826231.032964:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:d203faaf-7906-4d13-b1ba-8437467d6639:2351:1874670011361280:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826231.037057:0:2351:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d850a3000: committing for last_committed 8589942878 gen 1 00000100:00100000:2.0:1787826231.037094:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:d203faaf-7906-4d13-b1ba-8437467d6639:2351:1874670011361152:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826231.037110:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:d203faaf-7906-4d13-b1ba-8437467d6639:2351:1874670011361408:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826231.037131:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011360768:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826231.037142:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011361024:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826231.037152:0:2351:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff907d850a3000: committing for last_committed 21474840470 gen 1 00000100:00100000:2.0:1787826231.037165:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:d203faaf-7906-4d13-b1ba-8437467d6639:2351:1874670011361280:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826231.037175:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011360640:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826231.037186:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011360896:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826231.144803:0:63969:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:63969:1874670011361536:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826231.144833:0:63969:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.144845:0:63969:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.144861:0:63969:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.144872:0:63969:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.144885:0:63969:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.144893:0:63969:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.144900:0:63969:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826231.144911:0:63969:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d908a2d00 x1874670011361536/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/688 e 0 to 0 dl 1787826249 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1787826231.144925:0:63969:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.144938:0:63969:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1787826231.144945:0:63969:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1787826231.144976:0:63969:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:3.0:1787826231.144982:0:63969:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b452 00000080:00010000:3.0:1787826231.144988:0:63969:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826231.144990:0:63969:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b452 from it ffff9dd140d7fbf8 00010000:00010000:3.0:1787826231.144994:0:63969:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.145002:0:63969:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787826231.145025:0:63969:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826231.145050:0:63969:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f200c.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), file ffff907d83cc2300, open_flags 18800, mode 0 opened 0 00000080:00200000:3.0:1787826231.145057:0:63969:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f200c.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=open 00000002:00010000:3.0:1787826231.145079:0:63969:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f200c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:3.0:1787826231.145124:0:63969:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b459 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 63969 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.145131:0:63969:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b459 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63969 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826231.145140:0:63969:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826231.145147:0:63969:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d835261c0 x1874670011361792/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826231.145156:0:63969:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b459 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63969 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826231.145175:0:63969:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:63969:1874670011361792:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826231.145333:0:63969:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da00390c0 going to sleep for 13 seconds 00000100:00100000:3.0:1787826231.157448:0:63969:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:63969:1874670011361792:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826231.157473:0:63969:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b459 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63969 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.157491:0:63969:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b459 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 63969 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.157498:0:63969:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b459 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 63969 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.157505:0:63969:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b459 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 63969 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.157513:0:63969:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b459 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 63969 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.157518:0:63969:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d98a19a00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826231.157522:0:63969:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b459 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 63969 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.157529:0:63969:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b459 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 63969 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.157537:0:63969:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b459 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 63969 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.157544:0:63969:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826231.157550:0:63969:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b459 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 63969 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.157557:0:63969:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b459 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 63969 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826231.157574:0:63969:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d835261c0 x1874670011361792/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826249 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826231.157593:0:63969:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:3.0:1787826231.157725:0:63969:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826231.157744:0:63969:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826231.157758:0:63969:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826231.157780:0:63969:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787826231.157787:0:63969:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826231.157794:0:63969:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826231.157798:0:63969:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b452 00000080:00010000:3.0:1787826231.157803:0:63969:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b452 from it ffff9dd140d7fbb0 00010000:00010000:3.0:1787826231.157806:0:63969:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826231.157814:0:63969:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787826231.157827:0:63969:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:3.0:1787826231.157834:0:63969:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00000080:00200080:3.0:1787826231.157859:0:63969:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=c0086915 arg=a903d0 00000100:00100000:3.0:1787826231.157911:0:63969:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:63969:1874670011361920:192.168.203.104@tcp:34:lfs.0 00000100:00100000:3.0:1787826231.157984:0:63969:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da00390c0 going to sleep for 13 seconds 00000100:00100000:3.0:1787826231.220111:0:63972:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:63972:1874670011362176:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826231.220134:0:63972:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b460 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63972 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.220147:0:63972:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b460 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 63972 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.220154:0:63972:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b460 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 63972 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.220161:0:63972:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b460 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 63972 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.220171:0:63972:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b460 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 63972 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.220177:0:63972:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883bc00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826231.220181:0:63972:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b460 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 63972 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.220188:0:63972:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b460 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 63972 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.220195:0:63972:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b460 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 63972 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.220202:0:63972:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826231.220210:0:63972:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b460 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 63972 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.220215:0:63972:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b460 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 63972 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826231.220236:0:63972:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908443c0 x1874670011362176/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826249 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826231.220254:0:63972:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:3.0:1787826231.220379:0:63972:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826231.220396:0:63972:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826231.220409:0:63972:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826231.220420:0:63972:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1787826231.220427:0:63972:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826231.220433:0:63972:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826231.220437:0:63972:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b452 00000080:00010000:3.0:1787826231.220442:0:63972:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b452 from it ffff9dd142097bb0 00010000:00010000:3.0:1787826231.220445:0:63972:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826231.220453:0:63972:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1787826231.220466:0:63972:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:3.0:1787826231.220474:0:63972:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00000080:00200080:3.0:1787826231.220502:0:63972:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=c0086915 arg=14943d0 00000100:00100000:3.0:1787826231.220686:0:63972:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:63972:1874670011362304:192.168.203.104@tcp:34:lfs.0 00000100:00100000:3.0:1787826231.221721:0:63972:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da0039600 going to sleep for 13 seconds 00000080:00200000:3.0:1787826231.291911:0:63981:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826231.291941:0:63981:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826231.291960:0:63981:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1787826231.291977:0:63981:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1787826231.291985:0:63981:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826231.291995:0:63981:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826231.291998:0:63981:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b452 00000080:00010000:3.0:1787826231.292006:0:63981:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b452 from it ffff9dd140d7fb58 00010000:00010000:3.0:1787826231.292011:0:63981:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1787826231.292020:0:63981:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1787826231.292034:0:63981:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826231.292046:0:63981:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f200c.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), flags=9 00000080:00200000:3.0:1787826231.292050:0:63981:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f200c.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=getattr 00000002:00010000:3.0:1787826231.292060:0:63981:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f200c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:3.0:1787826231.292102:0:63981:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b467 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 63981 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.292109:0:63981:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b467 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63981 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.292118:0:63981:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b467 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63981 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826231.292142:0:63981:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:63981:1874670011362432:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826231.292338:0:63981:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da0039240 going to sleep for 13 seconds 00000100:00100000:3.0:1787826231.300479:0:63981:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:63981:1874670011362432:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826231.300505:0:63981:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b467 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0efcb expref: -99 pid: 63981 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.300523:0:63981:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b467 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0efcb expref: -99 pid: 63981 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.300533:0:63981:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b467 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0efcb expref: -99 pid: 63981 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.300545:0:63981:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b467 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0efcb expref: -99 pid: 63981 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.306259:0:63981:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b467 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0efcb expref: -99 pid: 63981 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.306280:0:63981:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b467 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0efcb expref: -99 pid: 63981 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.306289:0:63981:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b467 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0efcb expref: -99 pid: 63981 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.306297:0:63981:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b467 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0efcb expref: -99 pid: 63981 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826231.306312:0:63981:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d83527840 x1874670011362432/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/848 e 0 to 0 dl 1787826249 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1787826231.306325:0:63981:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 264 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b467 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0efcb expref: -99 pid: 63981 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1787826231.306549:0:63981:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b467 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0efcb expref: -99 pid: 63981 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.306576:0:63981:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/501 (0 0) 00000080:00200000:3.0:1787826231.306583:0:63981:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1787826231.306641:0:63981:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x1f5:0x0](ffff907d9221e3c8): rc = 0 00000080:00200000:3.0:1787826231.306654:0:63981:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1f5:0x0]: layout version change: 2 -> 2 00000080:00010000:3.0:1787826231.306659:0:63981:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f5:0x0](ffff907d9221e3c8) for lock 0xe5f736940357b467 00000080:00010000:3.0:1787826231.306669:0:63981:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f5:0x0](ffff907d9221e3c8) 00000080:00010000:3.0:1787826231.306672:0:63981:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b467 from it ffff9dd140d7fb20 00010000:00010000:3.0:1787826231.306676:0:63981:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b467 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0efcb expref: -99 pid: 63981 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.306685:0:63981:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b467 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0efcb expref: -99 pid: 63981 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:3.0:1787826231.306737:0:63981:0:(file.c:6313:ll_getattr_dentry()) START file f200c.sanity-flr:[0x200000404:0x1f5:0x0](ffff907d9221e3c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787826231.306745:0:63981:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1f5:0x0](ffff907d9221e3c8),name=f200c.sanity-flr 00000002:00010000:3.0:1787826231.306773:0:63981:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f5:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826231.306784:0:63981:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b467 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0efcb expref: -99 pid: 63981 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826231.306794:0:63981:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b467 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0efcb expref: -99 pid: 63981 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787826231.306801:0:63981:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826231.306806:0:63981:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f5:0x0](ffff907d9221e3c8) 00000080:00010000:3.0:1787826231.306809:0:63981:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f5:0x0](ffff907d9221e3c8) for lock 0xe5f736940357b467 00000080:00010000:3.0:1787826231.306813:0:63981:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b467 from it ffff9dd140d7fcb8 00010000:00010000:3.0:1787826231.306816:0:63981:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b467 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0efcb expref: -99 pid: 63981 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826231.306824:0:63981:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19a00/0xe5f736940357b467 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f5:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0efcb expref: -99 pid: 63981 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787826231.306842:0:63981:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f5:0x0] ignore/verify layout 0/1, layout version 2 restore needed 0 00020000:00200000:3.0:1787826231.306853:0:63981:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1f5:0x0]: flr state: 2, move mirror from 1 to 1, have retried: 0, mirror count: 2 00020000:00200000:3.0:1787826231.306857:0:63981:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:3.0:1787826231.306860:0:63981:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1f5:0x0] 00000008:00010000:3.0:1787826231.306878:0:63981:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826231.306882:0:63981:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da0039270, osc lock ffff907d963b9c08, flags 0x400000400001002 00000008:00010000:3.0:1787826231.306888:0:63981:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826231.306889:0:63981:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da00392b0, osc lock ffff907d963b94b8, flags 0x400000400001002 00010000:00010000:3.0:1787826231.306932:0:63981:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008b44d461/0xe5f736940357b46e lrc: 3/1,0 mode: --/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.306943:0:63981:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008b44d461/0xe5f736940357b46e lrc: 3/1,0 mode: --/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.306953:0:63981:0:(ldlm_request.c:1139:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff907d83527840 x1874670011362560/t0(0) o101->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1787826231.307000:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011362560:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826231.307219:0:63981:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000004aad1e6f/0xe5f736940357b475 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.307230:0:63981:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000004aad1e6f/0xe5f736940357b475 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.307241:0:63981:0:(ldlm_request.c:1139:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff907d83524f00 x1874670011362688/t0(0) o101->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000020:00200000:3.0:1787826231.307258:0:63981:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000100:00100000:3.0:1787826231.307270:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011362688:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826231.315857:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826231.315919:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025470464 00000100:00100000:3.0:1787826231.315928:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025470464 00000100:00100000:3.0:1787826231.315982:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7099 00000100:00100000:3.0:1787826231.315991:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d835243c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:9562:x1874670025470464:12345-192.168.203.104@tcp:106: 00010000:00010000:3.0:1787826231.316010:0:40338:0:(ldlm_lockd.c:2525:ldlm_callback_handler()) ### glimpse ast ns: lustre-OST0001-osc-ffff907d850a3000 lock: 000000008408fc3d/0xe5f736940357b428 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0eedd expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826231.316024:0:40338:0:(ldlm_lockd.c:2116:ldlm_handle_gl_callback()) ### client glimpse AST callback handler ns: lustre-OST0001-osc-ffff907d850a3000 lock: 000000008408fc3d/0xe5f736940357b428 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0eedd expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:3.0:1787826231.316048:0:40338:0:(cl_object.c:322:cl_object_glimpse()) header@000000006c2d10ba[0x110, 1, [0x200000bd1:0x1:0x0] hash] 00000020:00010000:3.0:1787826231.316050:0:40338:0:(cl_object.c:322:cl_object_glimpse()) size: 0 mtime: 1787826221 atime: 1787826221 ctime: 1787826226 blocks: 0 00000020:00010000:3.0:1787826231.316054:0:40338:0:(cl_object.c:322:cl_object_glimpse()) header@00000000164fa231[0x116, 2, [0x280000400:0x303:0x0]] 00000020:00010000:3.0:1787826231.316056:0:40338:0:(cl_object.c:322:cl_object_glimpse()) size: 0 mtime: 1787826221 atime: 1787826221 ctime: 1787826226 blocks: 0 00000100:00100000:3.0:1787826231.316101:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d835243c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:9562:x1874670025470464:12345-192.168.203.104@tcp:106: Request processed in 111us (256us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826231.316109:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7099 00010000:00010000:3.0:1787826231.316229:0:2350:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008b44d461/0xe5f736940357b46e lrc: 6/2,0 mode: --/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.316244:0:2350:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008b44d461/0xe5f736940357b46e lrc: 6/2,0 mode: --/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0efd9 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.316257:0:2350:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008b44d461/0xe5f736940357b46e lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0efd9 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.316269:0:2350:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008b44d461/0xe5f736940357b46e lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0efd9 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:3.0:1787826231.316282:0:2350:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008b44d461/0xe5f736940357b46e lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0efd9 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:3.0:1787826231.316293:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826231.316297:0:2350:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008b44d461/0xe5f736940357b46e lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0efd9 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.316304:0:2350:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008b44d461/0xe5f736940357b46e lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0efd9 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.316313:0:2350:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008b44d461/0xe5f736940357b46e lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0efd9 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.316326:0:2350:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008b44d461/0xe5f736940357b46e lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0efd9 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1787826231.316344:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011362560:192.168.203.104@tcp:101:lfs.0 00010000:00010000:2.0:1787826231.325602:0:2351:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000004aad1e6f/0xe5f736940357b475 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826231.325655:0:2351:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000004aad1e6f/0xe5f736940357b475 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826231.325668:0:2351:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000004aad1e6f/0xe5f736940357b475 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826231.325678:0:2351:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000004aad1e6f/0xe5f736940357b475 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826231.325685:0:2351:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000004aad1e6f/0xe5f736940357b475 lrc: 5/1,0 mode: --/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826231.325699:0:2351:0:(osc_lock.c:192:osc_lock_lvb_update()) ### no dlm lock: acquired size 0, setting rss=0; leaving kms=0, end=18446744073709551615 00000020:00200000:2.0:1787826231.325707:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:2.0:1787826231.325718:0:2351:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000004aad1e6f/0xe5f736940357b475 lrc: 4/1,0 mode: --/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826231.325727:0:2351:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000004aad1e6f/0xe5f736940357b475 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826231.325735:0:2351:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000004aad1e6f/0xe5f736940357b475 lrc: 4/0,0 mode: --/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826231.325743:0:2351:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d98a19200 already unused, calling callback (ffffffffc118e410) 00010000:00010000:2.0:1787826231.325749:0:2351:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000004aad1e6f/0xe5f736940357b475 lrc: 5/0,0 mode: --/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826231.325759:0:2351:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1787826231.325776:0:2351:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000004aad1e6f/0xe5f736940357b475 lrc: 3/0,0 mode: --/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826231.325790:0:2351:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000004aad1e6f/0xe5f736940357b475 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826231.325813:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011362688:192.168.203.104@tcp:101:lfs.0 00000080:00200000:3.0:1787826231.325952:0:63981:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f5:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:3.0:1787826231.325957:0:63981:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98afec70 (ffff907da5608368/0) at cl_lock_release():245 00000020:00010000:3.0:1787826231.325961:0:63981:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98afec70 (ffff907da5608368/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826231.325967:0:63981:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da0039270 (ffff907d83f04878/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826231.325971:0:63981:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da00392b0 (ffff907d83f04488/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826231.325974:0:63981:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98afec70 (ffff907da5608368/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826231.325977:0:63981:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da0039270 (ffff907da5608368/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826231.325982:0:63981:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da00392b0 (ffff907da5608368/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826231.325995:0:63981:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f5:0x0] ignore/verify layout 0/1, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787826231.326042:0:63981:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826231.326060:0:63981:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826231.326074:0:63981:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1787826231.326086:0:63981:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1787826231.326093:0:63981:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826231.326099:0:63981:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826231.326103:0:63981:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b452 00000080:00010000:3.0:1787826231.326110:0:63981:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b452 from it ffff9dd140d7fbf8 00010000:00010000:3.0:1787826231.326113:0:63981:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1787826231.326122:0:63981:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b452 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ef5b expref: -99 pid: 63969 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1787826231.326134:0:63981:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826231.326140:0:63981:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f200c.sanity-flr, flags=265 00000080:00200000:3.0:1787826231.326145:0:63981:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1f5:0x0](ffff907d9221e3c8), inode mode 81a4 mask 44 00000080:00200008:3.0:1787826231.326151:0:63981:0:(file.c:1090:ll_file_open()) START file f200c.sanity-flr:[0x200000404:0x1f5:0x0](ffff907d9221e3c8), flags 154400 00000002:00010000:3.0:1787826231.326162:0:63981:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f5:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240114401 00010000:00010000:3.0:1787826231.326201:0:63981:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18800/0xe5f736940357b47c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 63981 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826231.326207:0:63981:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18800/0xe5f736940357b47c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63981 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826231.326217:0:63981:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826231.326225:0:63981:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83527840 x1874670011362944/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826231.326235:0:63981:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18800/0xe5f736940357b47c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63981 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826231.326253:0:63981:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:63981:1874670011362944:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826231.326399:0:63981:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da0039240 going to sleep for 13 seconds 00000100:00100000:2.0:1787826231.343841:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826231.343914:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025470592 00000100:00100000:3.0:1787826231.343922:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025470592 00000100:00100000:3.0:1787826231.343970:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7100 00000100:00100000:3.0:1787826231.343979:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d9114bc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:6934:x1874670025470592:12345-192.168.203.104@tcp:106: 00010000:00010000:3.0:1787826231.343993:0:50445:0:(ldlm_lockd.c:2525:ldlm_callback_handler()) ### glimpse ast ns: lustre-OST0001-osc-ffff907d850a3000 lock: 000000008408fc3d/0xe5f736940357b428 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0eedd expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826231.344005:0:50445:0:(ldlm_lockd.c:2116:ldlm_handle_gl_callback()) ### client glimpse AST callback handler ns: lustre-OST0001-osc-ffff907d850a3000 lock: 000000008408fc3d/0xe5f736940357b428 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0eedd expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:3.0:1787826231.344030:0:50445:0:(cl_object.c:322:cl_object_glimpse()) header@000000006c2d10ba[0x110, 1, [0x200000bd1:0x1:0x0] hash] 00000020:00010000:3.0:1787826231.344033:0:50445:0:(cl_object.c:322:cl_object_glimpse()) size: 0 mtime: 1787826221 atime: 1787826221 ctime: 1787826226 blocks: 0 00000020:00010000:3.0:1787826231.344037:0:50445:0:(cl_object.c:322:cl_object_glimpse()) header@00000000164fa231[0x116, 2, [0x280000400:0x303:0x0]] 00000020:00010000:3.0:1787826231.344038:0:50445:0:(cl_object.c:322:cl_object_glimpse()) size: 0 mtime: 1787826221 atime: 1787826221 ctime: 1787826226 blocks: 0 00000100:00100000:3.0:1787826231.344095:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d9114bc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:6934:x1874670025470592:12345-192.168.203.104@tcp:106: Request processed in 116us (261us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826231.344102:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7100 00000100:00100000:3.0:1787826231.350767:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826231.350892:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025470720 00000100:00100000:2.0:1787826231.350902:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025470720 00000100:00100000:2.0:1787826231.350992:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7101 00000100:00100000:2.0:1787826231.351001:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83525a40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:36729:x1874670025470720:12345-192.168.203.104@tcp:106: 00010000:00010000:2.0:1787826231.351015:0:40339:0:(ldlm_lockd.c:2525:ldlm_callback_handler()) ### glimpse ast ns: lustre-OST0001-osc-ffff907d850a3000 lock: 000000008408fc3d/0xe5f736940357b428 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0eedd expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826231.351028:0:40339:0:(ldlm_lockd.c:2116:ldlm_handle_gl_callback()) ### client glimpse AST callback handler ns: lustre-OST0001-osc-ffff907d850a3000 lock: 000000008408fc3d/0xe5f736940357b428 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0eedd expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1787826231.351053:0:40339:0:(cl_object.c:322:cl_object_glimpse()) header@000000006c2d10ba[0x110, 1, [0x200000bd1:0x1:0x0] hash] 00000020:00010000:2.0:1787826231.351055:0:40339:0:(cl_object.c:322:cl_object_glimpse()) size: 0 mtime: 1787826221 atime: 1787826221 ctime: 1787826226 blocks: 0 00000020:00010000:2.0:1787826231.351059:0:40339:0:(cl_object.c:322:cl_object_glimpse()) header@00000000164fa231[0x116, 2, [0x280000400:0x303:0x0]] 00000020:00010000:2.0:1787826231.351060:0:40339:0:(cl_object.c:322:cl_object_glimpse()) size: 0 mtime: 1787826221 atime: 1787826221 ctime: 1787826226 blocks: 0 00000100:00100000:2.0:1787826231.351114:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83525a40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:36729:x1874670025470720:12345-192.168.203.104@tcp:106: Request processed in 114us (363us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826231.351122:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7101 00000100:00100000:3.0:1787826231.364740:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826231.365504:0:50445:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:2.0:1787826231.365526:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025470848 00000100:00100000:2.0:1787826231.365536:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025470848 00000100:00100000:2.0:1787826231.365596:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7102 00000100:00100000:2.0:1787826231.365622:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d866b6c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:28337:x1874670025470848:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826231.365640:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d850a3000 lock: 00000000c925444c/0xe5f736940357b436 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x31d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x420000020000 nid: local remote: 0x7918213f4fc0eef9 expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826231.365722:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826231.365729:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d866b6c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:28337:x1874670025470848:12345-192.168.203.104@tcp:104: Request processed in 124us (999us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826231.365737:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7102 00010000:00010000:2.0:1787826234.233140:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826237.527192:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826237.530007:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025472000 00000100:00100000:3.0:1787826237.530022:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025472000 00000100:00100000:3.0:1787826237.530151:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7103 00000100:00100000:3.0:1787826237.530161:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91149a40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:6934:x1874670025472000:12345-192.168.203.104@tcp:106: 00010000:00010000:3.0:1787826237.530176:0:40339:0:(ldlm_lockd.c:2525:ldlm_callback_handler()) ### glimpse ast ns: lustre-OST0001-osc-ffff907d850a3000 lock: 000000008408fc3d/0xe5f736940357b428 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0eedd expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826237.530189:0:40339:0:(ldlm_lockd.c:2116:ldlm_handle_gl_callback()) ### client glimpse AST callback handler ns: lustre-OST0001-osc-ffff907d850a3000 lock: 000000008408fc3d/0xe5f736940357b428 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x303:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0eedd expref: -99 pid: 63688 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:3.0:1787826237.530218:0:40339:0:(cl_object.c:322:cl_object_glimpse()) header@000000006c2d10ba[0x110, 1, [0x200000bd1:0x1:0x0] hash] 00000020:00010000:3.0:1787826237.530220:0:40339:0:(cl_object.c:322:cl_object_glimpse()) size: 0 mtime: 1787826221 atime: 1787826221 ctime: 1787826226 blocks: 0 00000020:00010000:3.0:1787826237.530223:0:40339:0:(cl_object.c:322:cl_object_glimpse()) header@00000000164fa231[0x116, 2, [0x280000400:0x303:0x0]] 00000020:00010000:3.0:1787826237.530225:0:40339:0:(cl_object.c:322:cl_object_glimpse()) size: 0 mtime: 1787826221 atime: 1787826221 ctime: 1787826226 blocks: 0 00010000:00010000:3.0:1787826237.530307:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1787826237.530314:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91149a40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:6934:x1874670025472000:12345-192.168.203.104@tcp:106: Request processed in 156us (3132us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826237.530322:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7103 00000080:00200008:3.0:1787826237.841314:0:64411:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787826237.841329:0:64411:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826237.841408:0:64411:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826237.841471:0:64411:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b49f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 64411 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826237.841482:0:64411:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b49f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64411 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826237.841491:0:64411:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b49f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64411 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826237.841585:0:64411:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527c00 pname:cluuid:pid:xid:nid:opc:job lsof:6f891bcb-15cd-41e0-96e2-db843619c1be:64411:1874670011366016:192.168.203.104@tcp:101:lsof.0 00000100:00100000:3.0:1787826237.841708:0:64411:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d91709900 going to sleep for 13 seconds 00000080:00200008:3.0:1787826239.179400:0:64419:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d93310908), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787826239.179412:0:64419:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d93310908),name=/ 00000002:00010000:3.0:1787826239.179495:0:64419:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826239.179570:0:64419:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d83f0b000/0xe5f736940357b4a6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f057 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826239.179587:0:64419:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d83f0b000/0xe5f736940357b4a6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f057 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787826239.179595:0:64419:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826239.179603:0:64419:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d93310908) 00000080:00010000:3.0:1787826239.179735:0:64419:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d93310908) for lock 0xe5f736940357b4a6 00000080:00010000:3.0:1787826239.179744:0:64419:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b4a6 from it ffff9dd141e27cb8 00010000:00010000:3.0:1787826239.179749:0:64419:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d83f0b000/0xe5f736940357b4a6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f057 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826239.179758:0:64419:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d83f0b000/0xe5f736940357b4a6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f057 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200008:3.0:1787826239.179904:0:64419:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d93310908), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787826239.179910:0:64419:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d93310908),name=/ 00000002:00010000:3.0:1787826239.179917:0:64419:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826239.179924:0:64419:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d83f0b000/0xe5f736940357b4a6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f057 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826239.179932:0:64419:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d83f0b000/0xe5f736940357b4a6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f057 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787826239.179937:0:64419:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826239.179941:0:64419:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d93310908) 00000080:00010000:3.0:1787826239.179944:0:64419:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d93310908) for lock 0xe5f736940357b4a6 00000080:00010000:3.0:1787826239.179948:0:64419:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b4a6 from it ffff9dd141e27cb8 00010000:00010000:3.0:1787826239.179951:0:64419:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d83f0b000/0xe5f736940357b4a6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f057 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826239.179959:0:64419:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d850a3000 lock: ffff907d83f0b000/0xe5f736940357b4a6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f057 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1787826239.193690:0:37:0:(vvp_io.c:1889:vvp_io_init()) [0x200000bd1:0x1:0x0] ignore/verify layout 1/0, layout version 2 restore needed 0 00020000:00200000:3.0:1787826239.193717:0:37:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787826239.193753:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000bd1:0x1:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787826239.193767:0:37:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f5:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1787826239.193773:0:37:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1f5:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:3.0:1787826239.193776:0:37:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:3.0:1787826239.193778:0:37:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787826239.193784:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f5:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826239.193902:0:64419:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d93310908) 00000080:00200000:2.0:1787826239.193937:0:64419:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f5:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1787826239.193943:0:64419:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1f5:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826239.193945:0:64419:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1787826239.193947:0:64419:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1787826239.193957:0:64419:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f5:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826239.193961:0:64419:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200000404:0x1f5:0x0](ffff907d93322a08) 00000080:00200000:2.0:1787826239.194054:0:64419:0:(vvp_io.c:1889:vvp_io_init()) [0x200000bd1:0x1:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1787826239.194059:0:64419:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1787826239.194071:0:64419:0:(vvp_io.c:304:vvp_io_fini()) [0x200000bd1:0x1:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826239.194076:0:64419:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200000bd1:0x1:0x0](ffff907d92219148) 00000080:00200000:2.0:1787826239.194105:0:64419:0:(llite_lib.c:1688:ll_put_super()) VFS Op: cfg_instance lustre-client-ffff907d850a3000 (sb ffff907d850a3000) 00010000:00010000:2.0:1787826239.194145:0:64419:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.104@tcp lock: ffff907d98a18a00/0xe5f736940357b348 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x11000000000000 nid: local remote: 0x7918213f4fc0ecd0 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194156:0:64419:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.104@tcp lock: ffff907d98a18a00/0xe5f736940357b348 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1000000000000 nid: local remote: 0x7918213f4fc0ecd0 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194165:0:64419:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.104@tcp lock: ffff907d98a18a00/0xe5f736940357b348 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x11000400000000 nid: local remote: 0x7918213f4fc0ecd0 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194172:0:64419:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.203.104@tcp lock: ffff907d98a18a00/0xe5f736940357b348 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x11000400000000 nid: local remote: 0x7918213f4fc0ecd0 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194197:0:64419:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1787826239.194203:0:64419:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.104@tcp lock: ffff907d98a18e00/0xe5f736940357b372 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x7918213f4fc0ecd7 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194210:0:64419:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.104@tcp lock: ffff907d98a18e00/0xe5f736940357b372 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x7918213f4fc0ecd7 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194216:0:64419:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.104@tcp lock: ffff907d98a18e00/0xe5f736940357b372 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x11000400000000 nid: local remote: 0x7918213f4fc0ecd7 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194222:0:64419:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.203.104@tcp lock: ffff907d98a18e00/0xe5f736940357b372 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x11000400000000 nid: local remote: 0x7918213f4fc0ecd7 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194247:0:64419:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1787826239.194254:0:64419:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.104@tcp lock: ffff907d98a18600/0xe5f736940357b379 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x7918213f4fc0ecde expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194260:0:64419:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.104@tcp lock: ffff907d98a18600/0xe5f736940357b379 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x7918213f4fc0ecde expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194266:0:64419:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.104@tcp lock: ffff907d98a18600/0xe5f736940357b379 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x11000400000000 nid: local remote: 0x7918213f4fc0ecde expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194277:0:64419:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.203.104@tcp lock: ffff907d98a18600/0xe5f736940357b379 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x11000400000000 nid: local remote: 0x7918213f4fc0ecde expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194292:0:64419:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to regular blp list, 3 blwis in pool 10000000:01000000:2.0:1787826239.194296:0:64419:0:(mgc_request.c:547:config_log_end()) end config log lustre-client (0) 00010000:00010000:2.0:1787826239.194317:0:60472:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:2.0:1787826239.194324:0:60472:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.104@tcp lock: ffff907d98a18a00/0xe5f736940357b348 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1000400000000 nid: local remote: 0x7918213f4fc0ecd0 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194331:0:60472:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d98a18a00 already unused, calling callback (ffffffffc1319c20) 10000000:00010000:2.0:1787826239.194335:0:60472:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.104@tcp lock: ffff907d98a18a00/0xe5f736940357b348 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1000400000000 nid: local remote: 0x7918213f4fc0ecd0 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194343:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.104@tcp lock: ffff907d98a18a00/0xe5f736940357b348 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1008400000000 nid: local remote: 0x7918213f4fc0ecd0 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1787826239.194350:0:60472:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.104@tcp lock: ffff907d98a18a00/0xe5f736940357b348 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1009400000000 nid: local remote: 0x7918213f4fc0ecd0 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1787826239.194355:0:60472:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:01000000:2.0:1787826239.194358:0:60472:0:(mgc_request.c:944:mgc_blocking_ast()) log lustre-client: stopping, won't requeue 10000000:01000000:2.0:1787826239.194360:0:60472:0:(mgc_request.c:130:config_log_put()) dropping config log lustre-client 00010000:00010000:2.0:1787826239.194396:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.203.104@tcp lock: ffff907d98a18a00/0xe5f736940357b348 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x5809400000020 nid: local remote: 0x7918213f4fc0ecd0 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194402:0:60472:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1787826239.194425:0:60472:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.104@tcp lock: ffff907d98a18a00/0xe5f736940357b348 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x5809400000020 nid: local remote: 0x7918213f4fc0ecd0 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194433:0:60472:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.104@tcp lock: ffff907d98a18a00/0xe5f736940357b348 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x5809400000020 nid: local remote: 0x7918213f4fc0ecd0 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194445:0:60472:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1787826239.194448:0:60472:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.104@tcp lock: ffff907d98a18e00/0xe5f736940357b372 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x1000400000000 nid: local remote: 0x7918213f4fc0ecd7 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194454:0:60472:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d98a18e00 already unused, calling callback (ffffffffc1319c20) 10000000:00010000:2.0:1787826239.194457:0:60472:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.104@tcp lock: ffff907d98a18e00/0xe5f736940357b372 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x1000400000000 nid: local remote: 0x7918213f4fc0ecd7 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194465:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.104@tcp lock: ffff907d98a18e00/0xe5f736940357b372 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x1008400000000 nid: local remote: 0x7918213f4fc0ecd7 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1787826239.194484:0:60472:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.104@tcp lock: ffff907d98a18e00/0xe5f736940357b372 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x1009400000000 nid: local remote: 0x7918213f4fc0ecd7 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1787826239.194490:0:60472:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:01000000:2.0:1787826239.194491:0:60472:0:(mgc_request.c:944:mgc_blocking_ast()) log lustre-cliir: stopping, won't requeue 10000000:01000000:2.0:1787826239.194493:0:60472:0:(mgc_request.c:130:config_log_put()) dropping config log lustre-cliir 00010000:00010000:2.0:1787826239.194515:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.203.104@tcp lock: ffff907d98a18e00/0xe5f736940357b372 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0x7918213f4fc0ecd7 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194521:0:60472:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1787826239.194527:0:60472:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.104@tcp lock: ffff907d98a18e00/0xe5f736940357b372 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0x7918213f4fc0ecd7 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194533:0:60472:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.104@tcp lock: ffff907d98a18e00/0xe5f736940357b372 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0x7918213f4fc0ecd7 expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194542:0:60472:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1787826239.194545:0:60472:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.104@tcp lock: ffff907d98a18600/0xe5f736940357b379 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x1000400000000 nid: local remote: 0x7918213f4fc0ecde expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194552:0:60472:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d98a18600 already unused, calling callback (ffffffffc1319c20) 10000000:00010000:2.0:1787826239.194554:0:60472:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.104@tcp lock: ffff907d98a18600/0xe5f736940357b379 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x1000400000000 nid: local remote: 0x7918213f4fc0ecde expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194561:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.104@tcp lock: ffff907d98a18600/0xe5f736940357b379 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x1008400000000 nid: local remote: 0x7918213f4fc0ecde expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1787826239.194567:0:60472:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.104@tcp lock: ffff907d98a18600/0xe5f736940357b379 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x1009400000000 nid: local remote: 0x7918213f4fc0ecde expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1787826239.194572:0:60472:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 10000000:01000000:2.0:1787826239.194574:0:60472:0:(mgc_request.c:944:mgc_blocking_ast()) log params: stopping, won't requeue 10000000:01000000:2.0:1787826239.194576:0:60472:0:(mgc_request.c:130:config_log_put()) dropping config log params 00010000:00010000:2.0:1787826239.194588:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.203.104@tcp lock: ffff907d98a18600/0xe5f736940357b379 lrc: 2/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0x7918213f4fc0ecde expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826239.194593:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826239.194595:0:60472:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1787826239.194599:0:60472:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.104@tcp lock: ffff907d98a18600/0xe5f736940357b379 lrc: 1/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0x7918213f4fc0ecde expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194623:0:60472:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.104@tcp lock: ffff907d98a18600/0xe5f736940357b379 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0x7918213f4fc0ecde expref: -99 pid: 63508 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826239.194631:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826239.194633:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826239.194635:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826239.194650:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011366400:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1787826239.194650:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011366272:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1787826239.202435:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011366528:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1787826239.202531:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011366400:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1787826239.206588:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011366272:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1787826239.206625:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011366528:192.168.203.104@tcp:103:ldlm_bl.0 00000020:00000080:3.0:1787826239.208634:0:37:0:(genops.c:844:class_export_destroy()) destroying export ffff907d91f6e000/LOV_OSC_UUID for lustre-OST0001-osc-ffff907d850a3000 00000020:00000080:3.0:1787826239.210024:0:37:0:(genops.c:1183:obd_zombie_import_free()) destroying import ffff907d82ba7000 for lustre-OST0001-osc-ffff907d850a3000 00000020:00000080:3.0:1787826239.210032:0:37:0:(genops.c:885:class_export_put()) final put ffff907d86059800/d203faaf-7906-4d13-b1ba-8437467d6639 00000020:00000080:3.0:1787826239.210035:0:37:0:(genops.c:844:class_export_destroy()) destroying export ffff907d86059800/d203faaf-7906-4d13-b1ba-8437467d6639 for lustre-OST0001-osc-ffff907d850a3000 00000020:01000000:3.0:1787826239.210042:0:37:0:(genops.c:441:class_free_dev()) finishing cleanup of obd lustre-OST0001-osc-ffff907d850a3000 (d203faaf-7906-4d13-b1ba-8437467d6639) 00000020:00000080:3.0:1787826239.215126:0:37:0:(genops.c:844:class_export_destroy()) destroying export ffff907d91f6c800/LOV_OSC_UUID for lustre-OST0000-osc-ffff907d850a3000 00000020:00000080:3.0:1787826239.216948:0:37:0:(genops.c:1183:obd_zombie_import_free()) destroying import ffff907d86059000 for lustre-OST0000-osc-ffff907d850a3000 00000020:00000080:3.0:1787826239.216961:0:37:0:(genops.c:885:class_export_put()) final put ffff907d8605d800/d203faaf-7906-4d13-b1ba-8437467d6639 00000020:00000080:3.0:1787826239.216964:0:37:0:(genops.c:844:class_export_destroy()) destroying export ffff907d8605d800/d203faaf-7906-4d13-b1ba-8437467d6639 for lustre-OST0000-osc-ffff907d850a3000 00000020:01000000:3.0:1787826239.216971:0:37:0:(genops.c:441:class_free_dev()) finishing cleanup of obd lustre-OST0000-osc-ffff907d850a3000 (d203faaf-7906-4d13-b1ba-8437467d6639) 00000020:00000080:3.0:1787826239.217237:0:37:0:(genops.c:844:class_export_destroy()) destroying export ffff907d82ba4000/d203faaf-7906-4d13-b1ba-8437467d6639 for lustre-clilov-ffff907d850a3000 00000100:00100000:3.0:1787826239.222500:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d203faaf-7906-4d13-b1ba-8437467d6639:2347:1874670011367040:192.168.203.104@tcp:39:umount.0 00000100:00080000:3.0:1787826239.228561:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8605e000 lustre-MDT0000_UUID: changing import state from CONNECTING to CLOSED 00000100:00100000:3.0:1787826239.228831:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d203faaf-7906-4d13-b1ba-8437467d6639:2347:1874670011367040:192.168.203.104@tcp:39:umount.0 00000020:00000080:3.0:1787826239.232672:0:480:0:(genops.c:844:class_export_destroy()) destroying export ffff907d86054800/d203faaf-7906-4d13-b1ba-8437467d6639 for lustre-MDT0000-mdc-ffff907d850a3000 00000020:00000080:3.0:1787826239.232684:0:480:0:(genops.c:844:class_export_destroy()) destroying export ffff907d86054000/d203faaf-7906-4d13-b1ba-8437467d6639 for lustre-clilmv-ffff907d850a3000 00000020:00000080:3.0:1787826239.232690:0:480:0:(genops.c:885:class_export_put()) final put ffff907d8605a000/d203faaf-7906-4d13-b1ba-8437467d6639 00000020:00000080:3.0:1787826239.232692:0:480:0:(genops.c:844:class_export_destroy()) destroying export ffff907d8605a000/d203faaf-7906-4d13-b1ba-8437467d6639 for lustre-clilmv-ffff907d850a3000 00000020:01000000:3.0:1787826239.232696:0:480:0:(genops.c:441:class_free_dev()) finishing cleanup of obd lustre-clilmv-ffff907d850a3000 (d203faaf-7906-4d13-b1ba-8437467d6639) 00010000:00010000:2.0:1787826239.349047:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826241.263227:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826241.263355:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025473536 00000100:00100000:3.0:1787826241.263365:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025473536 00000100:00100000:3.0:1787826241.263434:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7104 00000100:00100000:3.0:1787826241.263445:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d911483c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:36728:x1874670025473536:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826241.263463:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008b44d461/0xe5f736940357b46e lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x31e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc0efd9 expref: -99 pid: 63981 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1787826241.263549:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826241.263556:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d911483c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:36728:x1874670025473536:12345-192.168.203.104@tcp:104: Request processed in 114us (337us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826241.263574:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7104 00010000:00010000:3.0:1787826241.402122:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826242.643717:0:64710:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64710:1874670011367680:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826242.643740:0:64710:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b4ad lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64710 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.643768:0:64710:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b4ad lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64710 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.643777:0:64710:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b4ad lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64710 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.643785:0:64710:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b4ad lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64710 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.643794:0:64710:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b4ad lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64710 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.643801:0:64710:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0a000 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826242.643806:0:64710:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b4ad lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 64710 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.643813:0:64710:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b4ad lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 64710 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.643821:0:64710:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b4ad lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 64710 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.643829:0:64710:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826242.643835:0:64710:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b4ad lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64710 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.643841:0:64710:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b4ad lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64710 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826242.643859:0:64710:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff907d908a3840 x1874670011367680/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 584/584 e 0 to 0 dl 1787826260 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826242.643873:0:64710:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:3.0:1787826242.643897:0:64710:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b49f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f049 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826242.643909:0:64710:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b49f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f049 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1787826242.643918:0:64710:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b49f from it ffff9dd141d53930 00010000:00010000:3.0:1787826242.643922:0:64710:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b49f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f049 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826242.643930:0:64710:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b49f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f049 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1787826242.643995:0:64710:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826242.644010:0:64710:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826242.644018:0:64710:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b49f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f049 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826242.644026:0:64710:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b49f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f049 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1787826242.644032:0:64710:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826242.644038:0:64710:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826242.644042:0:64710:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b49f 00000080:00010000:3.0:1787826242.644047:0:64710:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b49f from it ffff9dd141d53bf8 00010000:00010000:3.0:1787826242.644051:0:64710:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b49f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f049 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826242.644057:0:64710:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b49f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f049 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1787826242.644066:0:64710:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826242.644075:0:64710:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f202.sanity-flr, flags=777 00000080:00200000:3.0:1787826242.644080:0:64710:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826242.644086:0:64710:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826242.644093:0:64710:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b49f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f049 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1787826242.644101:0:64710:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b49f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f049 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1787826242.644107:0:64710:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826242.644111:0:64710:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826242.644114:0:64710:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b49f 00000080:00010000:3.0:1787826242.644118:0:64710:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b49f from it ffff9dd141d53b38 00010000:00010000:3.0:1787826242.644133:0:64710:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b49f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f049 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1787826242.644142:0:64710:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b49f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f049 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1787826242.644151:0:64710:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 3 00000080:00200000:3.0:1787826242.644157:0:64710:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), file ffff907d82cb5800, open_flags a141, mode 81b6 opened 0 00000080:00200000:3.0:1787826242.644167:0:64710:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=open|creat 00000002:00010000:3.0:1787826242.644187:0:64710:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f202.sanity-flr,[0x200000404:0x1f7:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0120502 00010000:00010000:3.0:1787826242.644200:0:64710:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b49f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc0f049 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1787826242.644242:0:64710:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b49f lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0f049 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1787826242.644250:0:64710:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787826242.644253:0:64710:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b49f lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0f049 expref: -99 pid: 64411 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1787826242.644287:0:64710:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18600/0xe5f736940357b4b4 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 64710 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.644295:0:64710:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18600/0xe5f736940357b4b4 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64710 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826242.644305:0:64710:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826242.644312:0:64710:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83524000 x1874670011367936/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 528/67896 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826242.644321:0:64710:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18600/0xe5f736940357b4b4 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64710 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826242.644340:0:64710:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64710:1874670011367936:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826242.646068:0:64710:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d82fb9000 going to sleep for 13 seconds 00000100:00100000:2.0:1787826242.695972:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826242.696196:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025473664 00000100:00100000:3.0:1787826242.696212:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025473664 00000100:00100000:3.0:1787826242.696300:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7105 00000100:00100000:3.0:1787826242.696312:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d9114ad00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32860:x1874670025473664:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826242.696336:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18600/0xe5f736940357b4b4 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0f09d expref: -99 pid: 64710 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826242.696531:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826242.696539:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d9114ad00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32860:x1874670025473664:12345-192.168.203.104@tcp:104: Request processed in 229us (576us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826242.696549:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7105 00000002:00100000:2.0:1787826242.722108:0:64710:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d83524000 x1874670011367936/t8589942888(8589942888) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 528/608 e 0 to 0 dl 1787826260 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826242.776733:0:64713:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826242.776774:0:64713:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826242.776860:0:64713:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 64713 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.776872:0:64713:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64713 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.776883:0:64713:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64713 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826242.776915:0:64713:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64713:1874670011368960:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826242.777060:0:64713:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d82fb9480 going to sleep for 13 seconds 00000100:00100000:3.0:1787826242.787735:0:64713:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64713:1874670011368960:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826242.787772:0:64713:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.787787:0:64713:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.787802:0:64713:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.787815:0:64713:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.787828:0:64713:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.787836:0:64713:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.787844:0:64713:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826242.787858:0:64713:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d835252c0 x1874670011368960/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/688 e 0 to 0 dl 1787826260 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1787826242.787874:0:64713:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.787891:0:64713:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1787826242.787899:0:64713:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1787826242.787935:0:64713:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:3.0:1787826242.787943:0:64713:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b4c2 00000080:00010000:3.0:1787826242.787962:0:64713:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826242.787966:0:64713:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b4c2 from it ffff9dd141e57bf8 00010000:00010000:3.0:1787826242.787969:0:64713:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.787978:0:64713:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787826242.788008:0:64713:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826242.788035:0:64713:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), file ffff907d84fb2480, open_flags 18800, mode 0 opened 0 00000080:00200000:3.0:1787826242.788043:0:64713:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=open 00000002:00010000:3.0:1787826242.788070:0:64713:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f202.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:3.0:1787826242.788130:0:64713:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4c9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64713 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.788138:0:64713:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4c9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64713 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826242.788151:0:64713:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826242.788158:0:64713:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d835252c0 x1874670011369216/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826242.788167:0:64713:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4c9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64713 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826242.788191:0:64713:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64713:1874670011369216:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826242.788434:0:64713:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d82fb9480 going to sleep for 13 seconds 00000100:00100000:3.0:1787826242.795711:0:64713:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64713:1874670011369216:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826242.795741:0:64713:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4c9 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64713 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.795762:0:64713:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4c9 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64713 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.795769:0:64713:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4c9 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64713 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.795777:0:64713:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4c9 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64713 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.795786:0:64713:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4c9 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64713 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.795793:0:64713:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d98a19000 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826242.795798:0:64713:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4c9 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64713 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.795805:0:64713:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4c9 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64713 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.795813:0:64713:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4c9 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 64713 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.795821:0:64713:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826242.795830:0:64713:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4c9 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64713 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.795836:0:64713:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4c9 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64713 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826242.795861:0:64713:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d835252c0 x1874670011369216/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/608 e 0 to 0 dl 1787826260 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826242.795882:0:64713:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:3.0:1787826242.796177:0:64713:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826242.796201:0:64713:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826242.796217:0:64713:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826242.796231:0:64713:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787826242.796238:0:64713:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826242.796244:0:64713:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826242.796249:0:64713:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b4c2 00000080:00010000:3.0:1787826242.796318:0:64713:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b4c2 from it ffff9dd141e57bb0 00010000:00010000:3.0:1787826242.796323:0:64713:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826242.796332:0:64713:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787826242.796350:0:64713:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:3.0:1787826242.796363:0:64713:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00000080:00200080:3.0:1787826242.796407:0:64713:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=c0086915 arg=1fc03d0 00000100:00100000:3.0:1787826242.796550:0:64713:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64713:1874670011369344:192.168.203.104@tcp:34:lfs.0 00000100:00100000:3.0:1787826242.796816:0:64713:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d82fb9480 going to sleep for 13 seconds 00000100:00100000:3.0:1787826242.816453:0:64713:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64713:1874670011369344:192.168.203.104@tcp:34:lfs.0 00000080:00200008:3.0:1787826242.816852:0:64713:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00000080:00200000:3.0:1787826242.837520:0:64717:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826242.837548:0:64717:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826242.837565:0:64717:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826242.837579:0:64717:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787826242.837587:0:64717:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826242.837595:0:64717:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826242.837598:0:64717:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b4c2 00000080:00010000:3.0:1787826242.837765:0:64717:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b4c2 from it ffff9dd141c37bf8 00010000:00010000:3.0:1787826242.837771:0:64717:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826242.837780:0:64717:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1787826242.837794:0:64717:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826242.837808:0:64717:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), file ffff907d82cbec00, open_flags 18800, mode 0 opened 0 00000080:00200000:3.0:1787826242.837814:0:64717:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=open 00000002:00010000:3.0:1787826242.837828:0:64717:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f202.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:3.0:1787826242.837878:0:64717:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4d0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64717 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.837886:0:64717:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4d0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64717 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826242.837897:0:64717:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826242.837909:0:64717:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d835252c0 x1874670011369600/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826242.837919:0:64717:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4d0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64717 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826242.837938:0:64717:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64717:1874670011369600:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826242.838145:0:64717:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d82fb9480 going to sleep for 13 seconds 00000100:00100000:3.0:1787826242.849381:0:64717:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64717:1874670011369600:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826242.849407:0:64717:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4d0 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64717 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.849420:0:64717:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4d0 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64717 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.849427:0:64717:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4d0 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64717 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.849435:0:64717:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4d0 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64717 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.849444:0:64717:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4d0 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64717 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.849451:0:64717:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d98a19000 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826242.849455:0:64717:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4d0 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64717 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.849461:0:64717:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4d0 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64717 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.849532:0:64717:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4d0 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 64717 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.849541:0:64717:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826242.849548:0:64717:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4d0 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64717 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.849555:0:64717:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a19000/0xe5f736940357b4d0 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64717 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826242.849575:0:64717:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d835252c0 x1874670011369600/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/608 e 0 to 0 dl 1787826260 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826242.849588:0:64717:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:3.0:1787826242.849890:0:64717:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826242.849910:0:64717:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826242.849924:0:64717:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826242.849936:0:64717:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1787826242.849943:0:64717:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826242.849949:0:64717:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826242.849953:0:64717:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b4c2 00000080:00010000:3.0:1787826242.849960:0:64717:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b4c2 from it ffff9dd141c37bb0 00010000:00010000:3.0:1787826242.849964:0:64717:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826242.849977:0:64717:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1787826242.849991:0:64717:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:3.0:1787826242.850001:0:64717:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00000080:00200080:3.0:1787826242.850044:0:64717:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=c0086915 arg=ecf3d0 00000100:00100000:3.0:1787826242.850098:0:64717:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64717:1874670011369728:192.168.203.104@tcp:34:lfs.0 00000100:00100000:3.0:1787826242.850180:0:64717:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d82fb9480 going to sleep for 13 seconds 00000100:00100000:3.0:1787826242.856898:0:64717:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64717:1874670011369728:192.168.203.104@tcp:34:lfs.0 00000080:00200008:3.0:1787826242.857052:0:64717:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00010000:00010000:2.0:1787826242.894396:0:64718:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000009d88a315/0xe5f736940357b4de lrc: 3/1,0 mode: --/PR res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 64718 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826242.894410:0:64718:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000009d88a315/0xe5f736940357b4de lrc: 3/1,0 mode: --/PR res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 64718 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826242.894496:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011369984:192.168.203.104@tcp:101:lfs.0 00000020:00200000:2.0:1787826242.894630:0:64718:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826242.901241:0:2351:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000009d88a315/0xe5f736940357b4de lrc: 6/2,0 mode: --/PR res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 64718 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.901386:0:2351:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000009d88a315/0xe5f736940357b4de lrc: 6/2,0 mode: --/PR res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f13e expref: -99 pid: 64718 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.901402:0:2351:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000009d88a315/0xe5f736940357b4de lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f13e expref: -99 pid: 64718 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.901415:0:2351:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000009d88a315/0xe5f736940357b4de lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f13e expref: -99 pid: 64718 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:3.0:1787826242.901433:0:2351:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000009d88a315/0xe5f736940357b4de lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f13e expref: -99 pid: 64718 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:3.0:1787826242.901445:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:3.0:1787826242.901573:0:2351:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000009d88a315/0xe5f736940357b4de lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f13e expref: -99 pid: 64718 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.901583:0:2351:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000009d88a315/0xe5f736940357b4de lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f13e expref: -99 pid: 64718 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.901590:0:2351:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000009d88a315/0xe5f736940357b4de lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f13e expref: -99 pid: 64718 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826242.901598:0:2351:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000009d88a315/0xe5f736940357b4de lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f13e expref: -99 pid: 64718 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1787826242.901647:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011369984:192.168.203.104@tcp:101:lfs.0 00000080:00200000:2.0:1787826242.901780:0:64718:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f7:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1787826242.901787:0:64718:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da57929f8 (ffff907d884a7cf8/0) at cl_lock_release():245 00000020:00010000:2.0:1787826242.901919:0:64718:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da57929f8 (ffff907d884a7cf8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826242.901926:0:64718:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da032e1b0 (ffff907d884a7e18/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826242.901930:0:64718:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da57929f8 (ffff907d884a7cf8/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826242.901932:0:64718:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da032e1b0 (ffff907d884a7cf8/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826242.901945:0:64718:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f7:0x0] ignore/verify layout 0/1, layout version 1 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826242.902001:0:64718:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826242.902084:0:64718:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826242.902101:0:64718:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826242.902236:0:64718:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1787826242.902252:0:64718:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826242.902379:0:64718:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826242.902383:0:64718:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b4c2 00000080:00010000:2.0:1787826242.902389:0:64718:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b4c2 from it ffff9dd141e5fbf8 00010000:00010000:2.0:1787826242.902393:0:64718:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826242.902401:0:64718:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1787826242.902412:0:64718:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826242.902419:0:64718:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f202.sanity-flr, flags=265 00000080:00200000:2.0:1787826242.902424:0:64718:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1f7:0x0](ffff907d93313a88), inode mode 81a4 mask 44 00000080:00200008:2.0:1787826242.902431:0:64718:0:(file.c:1090:ll_file_open()) START file f202.sanity-flr:[0x200000404:0x1f7:0x0](ffff907d93313a88), flags 154400 00000002:00010000:2.0:1787826242.902446:0:64718:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f7:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240114401 00010000:00010000:2.0:1787826242.902561:0:64718:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a400/0xe5f736940357b4e5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64718 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826242.902569:0:64718:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a400/0xe5f736940357b4e5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64718 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826242.902581:0:64718:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826242.902590:0:64718:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114b480 x1874670011370240/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826242.902600:0:64718:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a400/0xe5f736940357b4e5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64718 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826242.902653:0:64718:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64718:1874670011370240:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826242.902996:0:64718:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da0039840 going to sleep for 13 seconds 00010000:00010000:2.0:1787826242.913985:0:60472:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1787826242.913993:0:60472:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a400/0xe5f736940357b4e5 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7918213f4fc0f14c expref: -99 pid: 64718 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826242.914000:0:60472:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907da563a400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:2.0:1787826242.914005:0:60472:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a400/0xe5f736940357b4e5 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f7:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7918213f4fc0f14c expref: -99 pid: 64718 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826242.914014:0:60472:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a400/0xe5f736940357b4e5 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f7:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7918213f4fc0f14c expref: -99 pid: 64718 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826242.914041:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a400/0xe5f736940357b4e5 lrc: 3/0,0 mode: PR/PR res: [0x200000404:0x1f7:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc0f14c expref: -99 pid: 64718 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826242.914058:0:60472:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b4d7 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f137 expref: -99 pid: 64718 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826242.914091:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a400/0xe5f736940357b4e5 lrc: 2/0,0 mode: --/PR res: [0x200000404:0x1f7:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0f14c expref: -99 pid: 64718 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826242.914098:0:60472:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1787826242.914124:0:60472:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a400/0xe5f736940357b4e5 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x1f7:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0f14c expref: -99 pid: 64718 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826242.914132:0:60472:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a400/0xe5f736940357b4e5 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x1f7:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0f14c expref: -99 pid: 64718 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826242.914145:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826242.914147:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826242.914149:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826242.914174:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011370368:192.168.203.104@tcp:103:ldlm_bl.0 00000080:00200000:2.0:1787826242.919804:0:64718:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1f7:0x0](ffff907d93313a88), inode mode 81a4 mask 4 00000080:00200000:2.0:1787826242.919831:0:64718:0:(xattr.c:760:ll_xattr_get()) VFS Op:inode=[0x200000404:0x1f7:0x0](ffff907d93313a88), xattr lov 00000080:00200008:2.0:1787826242.919871:0:64718:0:(file.c:6313:ll_getattr_dentry()) START file f202.sanity-flr:[0x200000404:0x1f7:0x0](ffff907d93313a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1787826242.919881:0:64718:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1f7:0x0](ffff907d93313a88),name=f202.sanity-flr 00000002:00010000:2.0:1787826242.919910:0:64718:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f7:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826242.919930:0:64718:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b4d7 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f137 expref: -99 pid: 64718 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826242.919944:0:64718:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b4d7 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f137 expref: -99 pid: 64718 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1787826242.919952:0:64718:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826242.919961:0:64718:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f7:0x0](ffff907d93313a88) 00000080:00010000:2.0:1787826242.919964:0:64718:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f7:0x0](ffff907d93313a88) for lock 0xe5f736940357b4d7 00000080:00010000:2.0:1787826242.919971:0:64718:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b4d7 from it ffff9dd141e5fcd8 00010000:00010000:2.0:1787826242.919975:0:64718:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b4d7 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f137 expref: -99 pid: 64718 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826242.919983:0:64718:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b4d7 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f137 expref: -99 pid: 64718 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1787826242.920011:0:64718:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f7:0x0] ignore/verify layout 0/1, layout version 1 restore needed 0 00000001:00010000:2.0:1787826242.920359:0:64718:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1f7:0x0] 00000008:00010000:2.0:1787826242.920438:0:64718:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826242.920443:0:64718:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322e30, osc lock ffff907d830a2d68, flags 0x400001002 00010000:00010000:2.0:1787826242.920460:0:64718:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000009d88a315/0xe5f736940357b4de lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x31f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f13e expref: -99 pid: 64718 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826242.920527:0:64718:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000009d88a315/0xe5f736940357b4de lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f13e expref: -99 pid: 64718 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1787826242.920540:0:64718:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826242.920543:0:64718:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000009d88a315/0xe5f736940357b4de lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f13e expref: -99 pid: 64718 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826242.920551:0:64718:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000009d88a315/0xe5f736940357b4de lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f13e expref: -99 pid: 64718 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1787826242.920561:0:64718:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826242.920570:0:64718:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f7:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1787826242.920574:0:64718:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da57929f8 (ffff907d884a7cf8/0) at cl_lock_release():245 00000020:00010000:2.0:1787826242.920577:0:64718:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da57929f8 (ffff907d884a7cf8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826242.920580:0:64718:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322e30 (ffff907d884a7e18/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826242.920584:0:64718:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da57929f8 (ffff907d884a7cf8/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826242.920587:0:64718:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322e30 (ffff907d884a7cf8/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826242.920597:0:64718:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f7:0x0] ignore/verify layout 0/1, layout version 1 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826242.920659:0:64718:0:(file.c:479:ll_file_release()) START file f202.sanity-flr:[0x200000404:0x1f7:0x0](ffff907d93313a88), flags 154000 00000002:00100000:2.0:1787826242.920720:0:64718:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d9114b480 x1874670011370240/t8589942892(8589942892) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 656/744 e 0 to 0 dl 1787826260 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1787826242.920731:0:64718:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1787826242.920736:0:64718:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d91148f00 x1874670011370624/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1787826242.920758:0:64718:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64718:1874670011370624:192.168.203.104@tcp:35:lfs.0 00000100:00100000:2.0:1787826242.920943:0:64718:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da0039cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1787826242.923361:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011370368:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1787826242.931654:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826242.932524:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025473792 00000100:00100000:3.0:1787826242.932535:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025473792 00000100:00100000:3.0:1787826242.932656:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7106 00000100:00100000:3.0:1787826242.932666:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91148b40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32458:x1874670025473792:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826242.932692:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b4d7 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0f137 expref: -99 pid: 64718 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1787826242.932773:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826242.932779:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91148b40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32458:x1874670025473792:12345-192.168.203.104@tcp:104: Request processed in 116us (1134us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826242.932789:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7106 00000100:00100000:2.0:1787826242.942139:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826242.942795:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025473920 00000100:00100000:3.0:1787826242.942807:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025473920 00000100:00100000:3.0:1787826242.942908:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7107 00000100:00100000:3.0:1787826242.942920:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91149e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32860:x1874670025473920:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826242.942937:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b4ec lrc: 2/0,0 mode: CW/CW res: [0x200000404:0x1f7:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x7918213f4fc0f161 expref: -99 pid: 64718 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826242.943024:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826242.943031:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91149e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32860:x1874670025473920:12345-192.168.203.104@tcp:104: Request processed in 115us (902us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826242.943038:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7107 00000100:00100000:2.0:1787826242.961039:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826242.961127:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025474048 00000100:00100000:3.0:1787826242.961137:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025474048 00000100:00100000:3.0:1787826242.961195:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7108 00000100:00100000:3.0:1787826242.961205:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91148000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32456:x1874670025474048:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826242.961224:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0aa00/0xe5f736940357b4f3 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0f176 expref: -99 pid: 64718 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826242.961305:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826242.961311:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91148000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32456:x1874670025474048:12345-192.168.203.104@tcp:104: Request processed in 109us (284us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826242.961319:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7108 00000080:00200000:2.0:1787826243.015170:0:64720:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826243.015199:0:64720:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826243.015217:0:64720:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1787826243.015233:0:64720:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1787826243.015242:0:64720:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826243.015250:0:64720:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826243.017354:0:64720:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b4c2 00000080:00010000:2.0:1787826243.017367:0:64720:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b4c2 from it ffff9dd141d5bbf8 00010000:00010000:2.0:1787826243.017373:0:64720:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1787826243.017393:0:64720:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1787826243.017409:0:64720:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826243.017424:0:64720:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), file ffff907d91738c00, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1787826243.017431:0:64720:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=open 00000002:00010000:2.0:1787826243.017452:0:64720:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f202.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:2.0:1787826243.018749:0:64720:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a200/0xe5f736940357b4fa lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64720 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826243.018764:0:64720:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a200/0xe5f736940357b4fa lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64720 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826243.018776:0:64720:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826243.018787:0:64720:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d91148780 x1874670011372160/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826243.018797:0:64720:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a200/0xe5f736940357b4fa lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64720 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826243.018819:0:64720:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64720:1874670011372160:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826243.018996:0:64720:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da0039600 going to sleep for 13 seconds 00000080:00200000:2.0:1787826243.081971:0:64724:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826243.081997:0:64724:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826243.082014:0:64724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1787826243.082036:0:64724:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1787826243.082043:0:64724:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826243.082050:0:64724:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826243.082053:0:64724:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b4c2 00000080:00010000:2.0:1787826243.082060:0:64724:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b4c2 from it ffff9dd141d5bbf8 00010000:00010000:2.0:1787826243.082064:0:64724:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1787826243.082074:0:64724:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:2.0:1787826243.082088:0:64724:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826243.082101:0:64724:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), file ffff907d91ede600, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1787826243.082108:0:64724:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=open 00000002:00010000:2.0:1787826243.082121:0:64724:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f202.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:2.0:1787826243.082169:0:64724:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563bc00/0xe5f736940357b501 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64724 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826243.082177:0:64724:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563bc00/0xe5f736940357b501 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64724 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826243.082188:0:64724:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826243.082199:0:64724:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114a1c0 x1874670011372544/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826243.082215:0:64724:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563bc00/0xe5f736940357b501 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64724 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826243.082236:0:64724:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64724:1874670011372544:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826243.082574:0:64724:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da0039e40 going to sleep for 13 seconds 00000080:00200000:2.0:1787826243.132349:0:64726:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826243.132377:0:64726:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826243.132395:0:64726:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1787826243.132411:0:64726:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:2.0:1787826243.132419:0:64726:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826243.132427:0:64726:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826243.132430:0:64726:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b4c2 00000080:00010000:2.0:1787826243.132438:0:64726:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b4c2 from it ffff9dd141e5fbf8 00010000:00010000:2.0:1787826243.132442:0:64726:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1787826243.132455:0:64726:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d98a18a00/0xe5f736940357b4c2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f0c7 expref: -99 pid: 64713 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:2.0:1787826243.132535:0:64726:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826243.132551:0:64726:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), file ffff907d91738780, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1787826243.132559:0:64726:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=open 00000002:00010000:2.0:1787826243.132574:0:64726:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f202.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:2.0:1787826243.132890:0:64726:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ba00/0xe5f736940357b508 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64726 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826243.132898:0:64726:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ba00/0xe5f736940357b508 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64726 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826243.132909:0:64726:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826243.132921:0:64726:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d911492c0 x1874670011372928/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826243.132930:0:64726:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ba00/0xe5f736940357b508 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64726 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826243.132952:0:64726:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64726:1874670011372928:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826243.133179:0:64726:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da00399c0 going to sleep for 13 seconds 00000100:00100000:3.0:1787826243.224916:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826243.225073:0:50445:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:2.0:1787826243.225093:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025474176 00000100:00100000:2.0:1787826243.225101:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025474176 00000100:00100000:2.0:1787826243.225152:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7109 00000100:00100000:2.0:1787826243.225163:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d84e74050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:27403:x1874670025474176:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826243.225183:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000009d88a315/0xe5f736940357b4de lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x31f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc0f13e expref: -99 pid: 64718 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826243.225273:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826243.225280:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d84e74050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:27403:x1874670025474176:12345-192.168.203.104@tcp:104: Request processed in 119us (374us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826243.225287:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7109 00000100:00100000:2.0:1787826243.235246:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826243.235343:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025474304 00000100:00100000:3.0:1787826243.235353:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025474304 00000100:00100000:3.0:1787826243.235411:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7110 00000100:00100000:3.0:1787826243.235421:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91149a40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:28563:x1874670025474304:12345-192.168.203.104@tcp:105: 00010000:00010000:3.0:1787826243.235437:0:40339:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000066987f3b/0xe5f736940357b524 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f241 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826243.235450:0:40339:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000066987f3b/0xe5f736940357b524 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f241 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826243.235459:0:40339:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000066987f3b/0xe5f736940357b524 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f241 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826243.235480:0:40339:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000066987f3b/0xe5f736940357b524 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10080000000000 nid: local remote: 0x7918213f4fc0f241 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826243.235489:0:40339:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000066987f3b/0xe5f736940357b524 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10080000000000 nid: local remote: 0x7918213f4fc0f241 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826243.235499:0:40339:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000066987f3b/0xe5f736940357b524 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x80000000000 nid: local remote: 0x7918213f4fc0f241 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826243.235516:0:40339:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff907d83f0b600) 00000100:00100000:3.0:1787826243.235572:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91149a40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:28563:x1874670025474304:12345-192.168.203.104@tcp:105: Request processed in 152us (334us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826243.235580:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7110 00000080:00200000:2.0:1787826243.335195:0:64727:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f7:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1787826243.337033:0:64727:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b200/0xe5f736940357b52b lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1f7:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f24f expref: -99 pid: 64727 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826243.337056:0:64727:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b200/0xe5f736940357b52b lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1f7:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f24f expref: -99 pid: 64727 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1787826243.337068:0:64727:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x1f7:0x0](000000008a489d5a) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b200/0xe5f736940357b52b lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f7:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f24f expref: -99 pid: 64727 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1787826243.337113:0:64727:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f7:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1787826243.337124:0:64727:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1787826243.337133:0:64727:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:2.0:1787826243.337151:0:64727:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f7:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:2.0:1787826243.337167:0:64727:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 1048576 blocks 2048 [cma]time 1787826243 1787826243 0 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000066987f3b/0xe5f736940357b524 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x31f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0f241 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1787826243.337238:0:64727:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1f7:0x0]: layout version change: 4294967294 -> 3 00010000:00010000:2.0:1787826243.337245:0:64727:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b200/0xe5f736940357b52b lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f7:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f24f expref: -99 pid: 64727 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826243.337265:0:64727:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b200/0xe5f736940357b52b lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f7:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f24f expref: -99 pid: 64727 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1787826243.337285:0:64727:0:(file.c:2170:ll_file_io_generic()) Range lock [256, 511] 00020000:00200000:2.0:1787826243.337292:0:64727:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 0 chunk: [1048576, 2097152) 2097152, 1048576 00020000:00200000:2.0:1787826243.337295:0:64727:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00000080:00200000:2.0:1787826243.337307:0:64727:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [256, 511] 00000008:00010000:2.0:1787826243.337318:0:64727:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1787826243.337323:0:64727:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322e30, osc lock ffff907d830a2d68, flags 0x0 00010000:00010000:2.0:1787826243.337373:0:64727:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826243.337382:0:64727:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [1048576->2097151] (req 1048576->2097151) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826243.337393:0:64727:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [1048576->2097151] (req 1048576->2097151) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826243.337419:0:64727:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:64727:1874670011374720:192.168.203.104@tcp:101:dd.0 00000100:00100000:2.0:1787826243.337520:0:64727:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da0039e40 going to sleep for 11 seconds 00000080:00200000:1.0:1787826243.363025:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@000000008fdb4c7a 461: 1< 0 - - > 2< 1888256 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.363028:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000131b1477 00000080:00200000:1.0:1787826243.363029:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 206. 00000080:00200000:1.0:1787826243.363031:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 462 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.363045:0:64727:0:(rw26.c:875:ll_write_end()) pos 1892352, len 4096, copied 4096 00000080:00200000:1.0:1787826243.363049:0:64727:0:(rw26.c:933:ll_write_end()) page@000000009d9b8d6b[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 462] 00000080:00200000:1.0:1787826243.363052:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @0000000002df503f 17ffffc0001001 3:0 ffff907da560a2b8 462 lru 00000080:00200000:1.0:1787826243.363058:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000af30c5b9 462: 1< 0 - - > 2< 1892352 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.363061:0:64727:0:(rw26.c:933:ll_write_end()) end page@000000009d9b8d6b 00000080:00200000:1.0:1787826243.363062:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 207. 00000080:00200000:1.0:1787826243.363064:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 463 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.363077:0:64727:0:(rw26.c:875:ll_write_end()) pos 1896448, len 4096, copied 4096 00000080:00200000:1.0:1787826243.363082:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000e822f64b[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 463] 00000080:00200000:1.0:1787826243.363084:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @000000008e4be142 17ffffc0001001 3:0 ffff907da560a000 463 lru 00000080:00200000:1.0:1787826243.363090:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@000000000dce75cf 463: 1< 0 - - > 2< 1896448 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.363092:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000e822f64b 00000080:00200000:1.0:1787826243.363093:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 208. 00000080:00200000:1.0:1787826243.363095:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 464 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.363108:0:64727:0:(rw26.c:875:ll_write_end()) pos 1900544, len 4096, copied 4096 00000080:00200000:1.0:1787826243.363112:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000ad9bf175[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 464] 00000080:00200000:1.0:1787826243.363115:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @000000006c7eb17c 17ffffc0001001 3:0 ffff907da560aae0 464 lru 00000080:00200000:1.0:1787826243.363120:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000b48953f0 464: 1< 0 - - > 2< 1900544 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.363122:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000ad9bf175 00000080:00200000:1.0:1787826243.363123:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 209. 00000080:00200000:1.0:1787826243.363125:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 465 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.363138:0:64727:0:(rw26.c:875:ll_write_end()) pos 1904640, len 4096, copied 4096 00000080:00200000:1.0:1787826243.363142:0:64727:0:(rw26.c:933:ll_write_end()) page@000000009916a98e[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 465] 00000080:00200000:1.0:1787826243.363145:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000916b0d36 17ffffc0001001 3:0 ffff907da560ae80 465 lru 00000080:00200000:1.0:1787826243.363150:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000e768b280 465: 1< 0 - - > 2< 1904640 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.363152:0:64727:0:(rw26.c:933:ll_write_end()) end page@000000009916a98e 00000080:00200000:1.0:1787826243.363153:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 210. 00000080:00200000:1.0:1787826243.363156:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 466 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.363170:0:64727:0:(rw26.c:875:ll_write_end()) pos 1908736, len 4096, copied 4096 00000080:00200000:1.0:1787826243.363174:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000ec1008a9[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 466] 00000080:00200000:1.0:1787826243.363177:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000e5d77dbf 17ffffc0001001 3:0 ffff907da560a3a0 466 lru 00000080:00200000:1.0:1787826243.363182:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@000000007787cce4 466: 1< 0 - - > 2< 1908736 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.363184:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000ec1008a9 00000080:00200000:1.0:1787826243.363185:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 211. 00000080:00200000:1.0:1787826243.363187:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 467 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.363202:0:64727:0:(rw26.c:875:ll_write_end()) pos 1912832, len 4096, copied 4096 00000080:00200000:1.0:1787826243.363206:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000c348d0e9[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 467] 00000080:00200000:1.0:1787826243.363209:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000c0b21eab 17ffffc0001001 3:0 ffff907da560a488 467 lru 00000080:00200000:1.0:1787826243.363214:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000d314a091 467: 1< 0 - - > 2< 1912832 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.363216:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000c348d0e9 00000080:00200000:1.0:1787826243.363217:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 212. 00000080:00200000:1.0:1787826243.363218:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 468 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.363231:0:64727:0:(rw26.c:875:ll_write_end()) pos 1916928, len 4096, copied 4096 00000080:00200000:1.0:1787826243.363235:0:64727:0:(rw26.c:933:ll_write_end()) page@0000000065a54c68[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 468] 00000080:00200000:1.0:1787826243.363238:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000df43668b 17ffffc0001001 3:0 ffff907da560a740 468 lru 00000080:00200000:1.0:1787826243.363243:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000a1f932ee 468: 1< 0 - - > 2< 1916928 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.363245:0:64727:0:(rw26.c:933:ll_write_end()) end page@0000000065a54c68 00000080:00200000:1.0:1787826243.363246:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 213. 00000080:00200000:1.0:1787826243.363248:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 469 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.365263:0:64727:0:(rw26.c:875:ll_write_end()) pos 1921024, len 4096, copied 4096 00000080:00200000:1.0:1787826243.365280:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000bfb54600[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 469] 00000080:00200000:1.0:1787826243.365284:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000764a3772 17ffffc0001001 3:0 ffff907da560abc8 469 lru 00000080:00200000:1.0:1787826243.365292:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000160bbde8 469: 1< 0 - - > 2< 1921024 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.365294:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000bfb54600 00000080:00200000:1.0:1787826243.365295:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 214. 00000080:00200000:1.0:1787826243.365299:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 470 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.365330:0:64727:0:(rw26.c:875:ll_write_end()) pos 1925120, len 4096, copied 4096 00000080:00200000:1.0:1787826243.365335:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000856b5d3d[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 470] 00000080:00200000:1.0:1787826243.365337:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000ef2ea1bc 17ffffc0001001 3:0 ffff907da560acb0 470 lru 00000080:00200000:1.0:1787826243.365344:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000624f36f0 470: 1< 0 - - > 2< 1925120 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.365346:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000856b5d3d 00000080:00200000:1.0:1787826243.365347:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 215. 00000080:00200000:1.0:1787826243.365349:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 471 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.365363:0:64727:0:(rw26.c:875:ll_write_end()) pos 1929216, len 4096, copied 4096 00000080:00200000:1.0:1787826243.365367:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000d069d63a[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 471] 00000080:00200000:1.0:1787826243.365370:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000be8b24f3 17ffffc0001001 3:0 ffff907da560a9f8 471 lru 00000080:00200000:1.0:1787826243.365375:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000a8cc6ab7 471: 1< 0 - - > 2< 1929216 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.365387:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000d069d63a 00000080:00200000:1.0:1787826243.365388:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 216. 00000080:00200000:1.0:1787826243.365390:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 472 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.365405:0:64727:0:(rw26.c:875:ll_write_end()) pos 1933312, len 4096, copied 4096 00000080:00200000:1.0:1787826243.365409:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000915c2864[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 472] 00000080:00200000:1.0:1787826243.365411:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @000000003357f093 17ffffc0001001 3:0 ffff907da560a570 472 lru 00000080:00200000:1.0:1787826243.365416:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@000000005ffa9250 472: 1< 0 - - > 2< 1933312 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.365418:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000915c2864 00000080:00200000:1.0:1787826243.365419:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 217. 00000080:00200000:1.0:1787826243.365421:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 473 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.365435:0:64727:0:(rw26.c:875:ll_write_end()) pos 1937408, len 4096, copied 4096 00000080:00200000:1.0:1787826243.365439:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000c07620f7[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 473] 00000080:00200000:1.0:1787826243.365442:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000e05497da 17ffffc0001001 3:0 ffff907da560a828 473 lru 00000080:00200000:1.0:1787826243.365447:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@0000000007ba3d07 473: 1< 0 - - > 2< 1937408 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.365449:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000c07620f7 00000080:00200000:1.0:1787826243.365450:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 218. 00000080:00200000:1.0:1787826243.365452:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 474 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.365469:0:64727:0:(rw26.c:875:ll_write_end()) pos 1941504, len 4096, copied 4096 00000080:00200000:1.0:1787826243.365473:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000e20469ba[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 474] 00000080:00200000:1.0:1787826243.365475:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @000000001cb01f3a 17ffffc0001001 3:0 ffff907da560ad98 474 lru 00000080:00200000:1.0:1787826243.365480:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@0000000032d54a25 474: 1< 0 - - > 2< 1941504 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.365482:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000e20469ba 00000080:00200000:1.0:1787826243.365483:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 219. 00000080:00200000:1.0:1787826243.365485:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 475 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.365506:0:64727:0:(rw26.c:875:ll_write_end()) pos 1945600, len 4096, copied 4096 00000080:00200000:1.0:1787826243.365510:0:64727:0:(rw26.c:933:ll_write_end()) page@0000000092e51061[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 475] 00000080:00200000:1.0:1787826243.365512:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000afb70675 17ffffc0001021 2:0 ffff907da560a658 475 lru 00000080:00200000:1.0:1787826243.365518:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@0000000043155d3e 475: 1< 0 - - > 2< 1945600 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.365520:0:64727:0:(rw26.c:933:ll_write_end()) end page@0000000092e51061 00000080:00200000:1.0:1787826243.365521:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 220. 00000080:00200000:1.0:1787826243.365523:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 476 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.365538:0:64727:0:(rw26.c:875:ll_write_end()) pos 1949696, len 4096, copied 4096 00000080:00200000:1.0:1787826243.365542:0:64727:0:(rw26.c:933:ll_write_end()) page@000000004961f7fc[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 476] 00000080:00200000:1.0:1787826243.365544:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000bb018955 17ffffc0001001 3:0 ffff907da560a0e8 476 lru 00000080:00200000:1.0:1787826243.365549:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000fce314f4 476: 1< 0 - - > 2< 1949696 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.365552:0:64727:0:(rw26.c:933:ll_write_end()) end page@000000004961f7fc 00000080:00200000:1.0:1787826243.365552:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 221. 00000080:00200000:1.0:1787826243.365554:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 477 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.365569:0:64727:0:(rw26.c:875:ll_write_end()) pos 1953792, len 4096, copied 4096 00000080:00200000:1.0:1787826243.365573:0:64727:0:(rw26.c:933:ll_write_end()) page@000000009e806e4a[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 477] 00000080:00200000:1.0:1787826243.365576:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000fc8dd879 17ffffc0001001 3:0 ffff907da63bb9f8 477 lru 00000080:00200000:1.0:1787826243.365581:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@0000000077f0d8d3 477: 1< 0 - - > 2< 1953792 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.365582:0:64727:0:(rw26.c:933:ll_write_end()) end page@000000009e806e4a 00000080:00200000:1.0:1787826243.365583:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 222. 00000080:00200000:1.0:1787826243.365585:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 478 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.365597:0:64727:0:(rw26.c:875:ll_write_end()) pos 1957888, len 4096, copied 4096 00000080:00200000:1.0:1787826243.365601:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000c61d6107[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 478] 00000080:00200000:1.0:1787826243.365604:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000ad16acb0 17ffffc0001001 3:0 ffff907da63bb570 478 lru 00000080:00200000:1.0:1787826243.365626:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000bd3aba19 478: 1< 0 - - > 2< 1957888 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.365628:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000c61d6107 00000080:00200000:1.0:1787826243.365629:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 223. 00000080:00200000:1.0:1787826243.365632:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 479 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.365648:0:64727:0:(rw26.c:875:ll_write_end()) pos 1961984, len 4096, copied 4096 00000080:00200000:1.0:1787826243.365652:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000d11539c6[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 479] 00000080:00200000:1.0:1787826243.365654:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @000000008b8211c9 17ffffc0001001 3:0 ffff907da63bb828 479 lru 00000080:00200000:1.0:1787826243.365659:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@000000009fcd960a 479: 1< 0 - - > 2< 1961984 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.365661:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000d11539c6 00000080:00200000:1.0:1787826243.365661:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 224. 00000080:00200000:1.0:1787826243.365663:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 480 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.365687:0:64727:0:(rw26.c:875:ll_write_end()) pos 1966080, len 4096, copied 4096 00000080:00200000:1.0:1787826243.365691:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000f226c7f5[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 480] 00000080:00200000:1.0:1787826243.365694:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000b02908d0 17ffffc0001001 3:0 ffff907da63bbd98 480 lru 00000080:00200000:1.0:1787826243.365698:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@0000000059568052 480: 1< 0 - - > 2< 1966080 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.365700:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000f226c7f5 00000080:00200000:1.0:1787826243.365701:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 225. 00000080:00200000:1.0:1787826243.365703:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 481 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.365717:0:64727:0:(rw26.c:875:ll_write_end()) pos 1970176, len 4096, copied 4096 00000080:00200000:1.0:1787826243.365721:0:64727:0:(rw26.c:933:ll_write_end()) page@000000007eb89b59[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 481] 00000080:00200000:1.0:1787826243.365724:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000fa0ea25a 17ffffc0001001 3:0 ffff907da63bb658 481 lru 00000080:00200000:1.0:1787826243.365728:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000d0aa028a 481: 1< 0 - - > 2< 1970176 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.365730:0:64727:0:(rw26.c:933:ll_write_end()) end page@000000007eb89b59 00000080:00200000:1.0:1787826243.365731:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 226. 00000080:00200000:1.0:1787826243.365732:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 482 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.365744:0:64727:0:(rw26.c:875:ll_write_end()) pos 1974272, len 4096, copied 4096 00000080:00200000:1.0:1787826243.365748:0:64727:0:(rw26.c:933:ll_write_end()) page@0000000077451aab[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 482] 00000080:00200000:1.0:1787826243.365751:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @0000000026fbee4d 17ffffc0001001 3:0 ffff907da63bb0e8 482 lru 00000080:00200000:1.0:1787826243.365756:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@000000003cf5d6a8 482: 1< 0 - - > 2< 1974272 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366045:0:64727:0:(rw26.c:933:ll_write_end()) end page@0000000077451aab 00000080:00200000:1.0:1787826243.366047:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 227. 00000080:00200000:1.0:1787826243.366049:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 483 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366066:0:64727:0:(rw26.c:875:ll_write_end()) pos 1978368, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366071:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000cdc1acf6[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 483] 00000080:00200000:1.0:1787826243.366074:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000e6a99d1c 17ffffc0001001 3:0 ffff907da63bb910 483 lru 00000080:00200000:1.0:1787826243.366079:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000e566c9e4 483: 1< 0 - - > 2< 1978368 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366083:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000cdc1acf6 00000080:00200000:1.0:1787826243.366084:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 228. 00000080:00200000:1.0:1787826243.366085:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 484 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366102:0:64727:0:(rw26.c:875:ll_write_end()) pos 1982464, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366106:0:64727:0:(rw26.c:933:ll_write_end()) page@0000000010fda41e[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 484] 00000080:00200000:1.0:1787826243.366109:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000a6ef6b0d 17ffffc0001001 3:0 ffff907da63bb1d0 484 lru 00000080:00200000:1.0:1787826243.366113:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@000000004f09aa37 484: 1< 0 - - > 2< 1982464 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366116:0:64727:0:(rw26.c:933:ll_write_end()) end page@0000000010fda41e 00000080:00200000:1.0:1787826243.366117:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 229. 00000080:00200000:1.0:1787826243.366119:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 485 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366131:0:64727:0:(rw26.c:875:ll_write_end()) pos 1986560, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366135:0:64727:0:(rw26.c:933:ll_write_end()) page@000000009616b487[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 485] 00000080:00200000:1.0:1787826243.366138:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @0000000026c610e4 17ffffc0001001 3:0 ffff907da63bb2b8 485 lru 00000080:00200000:1.0:1787826243.366143:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000aa4889bd 485: 1< 0 - - > 2< 1986560 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366145:0:64727:0:(rw26.c:933:ll_write_end()) end page@000000009616b487 00000080:00200000:1.0:1787826243.366146:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 230. 00000080:00200000:1.0:1787826243.366148:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 486 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366162:0:64727:0:(rw26.c:875:ll_write_end()) pos 1990656, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366167:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000085ff5d8[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 486] 00000080:00200000:1.0:1787826243.366169:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @000000002ca3ae7b 17ffffc0001001 3:0 ffff907da63bb000 486 lru 00000080:00200000:1.0:1787826243.366173:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@000000003b1c8653 486: 1< 0 - - > 2< 1990656 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366175:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000085ff5d8 00000080:00200000:1.0:1787826243.366176:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 231. 00000080:00200000:1.0:1787826243.366178:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 487 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366191:0:64727:0:(rw26.c:875:ll_write_end()) pos 1994752, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366195:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000e473cb33[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 487] 00000080:00200000:1.0:1787826243.366198:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @000000000f077f68 17ffffc0001001 3:0 ffff907da63bbae0 487 lru 00000080:00200000:1.0:1787826243.366202:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000570414ff 487: 1< 0 - - > 2< 1994752 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366204:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000e473cb33 00000080:00200000:1.0:1787826243.366205:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 232. 00000080:00200000:1.0:1787826243.366208:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 488 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366226:0:64727:0:(rw26.c:875:ll_write_end()) pos 1998848, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366230:0:64727:0:(rw26.c:933:ll_write_end()) page@000000009462a481[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 488] 00000080:00200000:1.0:1787826243.366233:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000b0197e40 17ffffc0001001 3:0 ffff907da63bbe80 488 lru 00000080:00200000:1.0:1787826243.366238:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@000000009ba29e88 488: 1< 0 - - > 2< 1998848 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366240:0:64727:0:(rw26.c:933:ll_write_end()) end page@000000009462a481 00000080:00200000:1.0:1787826243.366241:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 233. 00000080:00200000:1.0:1787826243.366243:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 489 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366256:0:64727:0:(rw26.c:875:ll_write_end()) pos 2002944, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366260:0:64727:0:(rw26.c:933:ll_write_end()) page@0000000014c69e89[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 489] 00000080:00200000:1.0:1787826243.366263:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000b8037155 17ffffc0001001 3:0 ffff907da63bb3a0 489 lru 00000080:00200000:1.0:1787826243.366268:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@000000009c2c93ca 489: 1< 0 - - > 2< 2002944 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366270:0:64727:0:(rw26.c:933:ll_write_end()) end page@0000000014c69e89 00000080:00200000:1.0:1787826243.366271:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 234. 00000080:00200000:1.0:1787826243.366273:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 490 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366293:0:64727:0:(rw26.c:875:ll_write_end()) pos 2007040, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366297:0:64727:0:(rw26.c:933:ll_write_end()) page@0000000042d6042b[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 490] 00000080:00200000:1.0:1787826243.366300:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000cba15e35 17ffffc0001021 2:0 ffff907da63bb488 490 lru 00000080:00200000:1.0:1787826243.366305:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@000000003680902d 490: 1< 0 - - > 2< 2007040 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366307:0:64727:0:(rw26.c:933:ll_write_end()) end page@0000000042d6042b 00000080:00200000:1.0:1787826243.366308:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 235. 00000080:00200000:1.0:1787826243.366310:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 491 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366322:0:64727:0:(rw26.c:875:ll_write_end()) pos 2011136, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366326:0:64727:0:(rw26.c:933:ll_write_end()) page@000000000ac762cd[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 491] 00000080:00200000:1.0:1787826243.366329:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @000000005682cb90 17ffffc0001001 3:0 ffff907da63bb740 491 lru 00000080:00200000:1.0:1787826243.366334:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@0000000029cfdb1c 491: 1< 0 - - > 2< 2011136 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366335:0:64727:0:(rw26.c:933:ll_write_end()) end page@000000000ac762cd 00000080:00200000:1.0:1787826243.366336:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 236. 00000080:00200000:1.0:1787826243.366338:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 492 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366351:0:64727:0:(rw26.c:875:ll_write_end()) pos 2015232, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366355:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000818c9061[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 492] 00000080:00200000:1.0:1787826243.366358:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000b2dabb64 17ffffc0001001 3:0 ffff907da63bbbc8 492 lru 00000080:00200000:1.0:1787826243.366363:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000d754f25a 492: 1< 0 - - > 2< 2015232 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366365:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000818c9061 00000080:00200000:1.0:1787826243.366366:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 237. 00000080:00200000:1.0:1787826243.366368:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 493 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366381:0:64727:0:(rw26.c:875:ll_write_end()) pos 2019328, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366386:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000de65c557[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 493] 00000080:00200000:1.0:1787826243.366388:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000e8f04ee1 17ffffc0001001 3:0 ffff907da63bbcb0 493 lru 00000080:00200000:1.0:1787826243.366393:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000a3267b55 493: 1< 0 - - > 2< 2019328 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366395:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000de65c557 00000080:00200000:1.0:1787826243.366396:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 238. 00000080:00200000:1.0:1787826243.366398:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 494 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366412:0:64727:0:(rw26.c:875:ll_write_end()) pos 2023424, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366417:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000d9f6ec07[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 494] 00000080:00200000:1.0:1787826243.366420:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @000000002e22a431 17ffffc0001001 3:0 ffff907d92bf4bc8 494 lru 00000080:00200000:1.0:1787826243.366425:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000803bce70 494: 1< 0 - - > 2< 2023424 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366427:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000d9f6ec07 00000080:00200000:1.0:1787826243.366428:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 239. 00000080:00200000:1.0:1787826243.366430:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 495 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366446:0:64727:0:(rw26.c:875:ll_write_end()) pos 2027520, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366450:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000ae4f96f3[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 495] 00000080:00200000:1.0:1787826243.366453:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @0000000008dc4cd7 17ffffc0001001 3:0 ffff907d92bf4cb0 495 lru 00000080:00200000:1.0:1787826243.366457:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000dfaeb3ba 495: 1< 0 - - > 2< 2027520 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366459:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000ae4f96f3 00000080:00200000:1.0:1787826243.366460:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 240. 00000080:00200000:1.0:1787826243.366462:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 496 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366473:0:64727:0:(rw26.c:875:ll_write_end()) pos 2031616, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366477:0:64727:0:(rw26.c:933:ll_write_end()) page@000000000c8dda5e[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 496] 00000080:00200000:1.0:1787826243.366480:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000ed8b2b65 17ffffc0001001 3:0 ffff907d92bf49f8 496 lru 00000080:00200000:1.0:1787826243.366486:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@000000000058fde9 496: 1< 0 - - > 2< 2031616 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366488:0:64727:0:(rw26.c:933:ll_write_end()) end page@000000000c8dda5e 00000080:00200000:1.0:1787826243.366489:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 241. 00000080:00200000:1.0:1787826243.366491:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 497 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366504:0:64727:0:(rw26.c:875:ll_write_end()) pos 2035712, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366508:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000d6b7de86[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 497] 00000080:00200000:1.0:1787826243.366510:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000009fb906 17ffffc0001001 3:0 ffff907d92bf4570 497 lru 00000080:00200000:1.0:1787826243.366515:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@0000000094a1e4fa 497: 1< 0 - - > 2< 2035712 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366517:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000d6b7de86 00000080:00200000:1.0:1787826243.366518:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 242. 00000080:00200000:1.0:1787826243.366520:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 498 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366534:0:64727:0:(rw26.c:875:ll_write_end()) pos 2039808, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366539:0:64727:0:(rw26.c:933:ll_write_end()) page@0000000017f18f75[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 498] 00000080:00200000:1.0:1787826243.366541:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000cd951d1a 17ffffc0001001 3:0 ffff907d92bf4828 498 lru 00000080:00200000:1.0:1787826243.366546:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@0000000014a521ab 498: 1< 0 - - > 2< 2039808 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366547:0:64727:0:(rw26.c:933:ll_write_end()) end page@0000000017f18f75 00000080:00200000:1.0:1787826243.366548:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 243. 00000080:00200000:1.0:1787826243.366550:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 499 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366562:0:64727:0:(rw26.c:875:ll_write_end()) pos 2043904, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366567:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000dcbd5914[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 499] 00000080:00200000:1.0:1787826243.366570:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000d83c5b54 17ffffc0001001 3:0 ffff907d92bf4d98 499 lru 00000080:00200000:1.0:1787826243.366574:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000e869e456 499: 1< 0 - - > 2< 2043904 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366576:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000dcbd5914 00000080:00200000:1.0:1787826243.366577:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 244. 00000080:00200000:1.0:1787826243.366579:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 500 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366593:0:64727:0:(rw26.c:875:ll_write_end()) pos 2048000, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366597:0:64727:0:(rw26.c:933:ll_write_end()) page@000000007201d404[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 500] 00000080:00200000:1.0:1787826243.366599:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000f43d40a6 17ffffc0001001 3:0 ffff907d92bf4658 500 lru 00000080:00200000:1.0:1787826243.366604:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@000000001c174e82 500: 1< 0 - - > 2< 2048000 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366647:0:64727:0:(rw26.c:933:ll_write_end()) end page@000000007201d404 00000080:00200000:1.0:1787826243.366648:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 245. 00000080:00200000:1.0:1787826243.366650:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 501 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366665:0:64727:0:(rw26.c:875:ll_write_end()) pos 2052096, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366669:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000701afc47[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 501] 00000080:00200000:1.0:1787826243.366672:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @0000000036f3cbe9 17ffffc0001001 3:0 ffff907d92bf40e8 501 lru 00000080:00200000:1.0:1787826243.366687:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000d78ae4d6 501: 1< 0 - - > 2< 2052096 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366690:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000701afc47 00000080:00200000:1.0:1787826243.366691:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 246. 00000080:00200000:1.0:1787826243.366692:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 502 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366709:0:64727:0:(rw26.c:875:ll_write_end()) pos 2056192, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366713:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000be4860aa[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 502] 00000080:00200000:1.0:1787826243.366716:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @000000003cbfd6da 17ffffc0001001 3:0 ffff907d92bf4910 502 lru 00000080:00200000:1.0:1787826243.366721:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@000000001a0927ff 502: 1< 0 - - > 2< 2056192 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366723:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000be4860aa 00000080:00200000:1.0:1787826243.366724:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 247. 00000080:00200000:1.0:1787826243.366726:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 503 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.366739:0:64727:0:(rw26.c:875:ll_write_end()) pos 2060288, len 4096, copied 4096 00000080:00200000:1.0:1787826243.366743:0:64727:0:(rw26.c:933:ll_write_end()) page@000000005ae6e653[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 503] 00000080:00200000:1.0:1787826243.366745:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000b1b5e853 17ffffc0001001 3:0 ffff907d92bf41d0 503 lru 00000080:00200000:1.0:1787826243.366750:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000dacdec4a 503: 1< 0 - - > 2< 2060288 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.366752:0:64727:0:(rw26.c:933:ll_write_end()) end page@000000005ae6e653 00000080:00200000:1.0:1787826243.366753:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 248. 00000080:00200000:1.0:1787826243.366755:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 504 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.368598:0:64727:0:(rw26.c:875:ll_write_end()) pos 2064384, len 4096, copied 4096 00000080:00200000:1.0:1787826243.368831:0:64727:0:(rw26.c:933:ll_write_end()) page@0000000055fa4c26[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 504] 00000080:00200000:1.0:1787826243.368835:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @000000009b2bd328 17ffffc0001001 3:0 ffff907d92bf42b8 504 lru 00000080:00200000:1.0:1787826243.368842:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000601ea7b8 504: 1< 0 - - > 2< 2064384 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.368845:0:64727:0:(rw26.c:933:ll_write_end()) end page@0000000055fa4c26 00000080:00200000:1.0:1787826243.368847:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 249. 00000080:00200000:1.0:1787826243.368851:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 505 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.368882:0:64727:0:(rw26.c:875:ll_write_end()) pos 2068480, len 4096, copied 4096 00000080:00200000:1.0:1787826243.368886:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000d0f677b6[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 505] 00000080:00200000:1.0:1787826243.368889:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @000000001a5f6583 17ffffc0001021 2:0 ffff907d92bf4000 505 lru 00000080:00200000:1.0:1787826243.368894:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@000000001355eb91 505: 1< 0 - - > 2< 2068480 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.368898:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000d0f677b6 00000080:00200000:1.0:1787826243.368899:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 250. 00000080:00200000:1.0:1787826243.368901:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 506 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.368914:0:64727:0:(rw26.c:875:ll_write_end()) pos 2072576, len 4096, copied 4096 00000080:00200000:1.0:1787826243.368918:0:64727:0:(rw26.c:933:ll_write_end()) page@000000000f65f7c2[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 506] 00000080:00200000:1.0:1787826243.368921:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @000000000a67da5c 17ffffc0001001 3:0 ffff907d92bf4ae0 506 lru 00000080:00200000:1.0:1787826243.368926:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@0000000047a08ae9 506: 1< 0 - - > 2< 2072576 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.368929:0:64727:0:(rw26.c:933:ll_write_end()) end page@000000000f65f7c2 00000080:00200000:1.0:1787826243.368930:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 251. 00000080:00200000:1.0:1787826243.368932:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 507 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.368946:0:64727:0:(rw26.c:875:ll_write_end()) pos 2076672, len 4096, copied 4096 00000080:00200000:1.0:1787826243.368951:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000dcaaf340[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 507] 00000080:00200000:1.0:1787826243.368953:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @000000007d5a769e 17ffffc0001001 3:0 ffff907d92bf4e80 507 lru 00000080:00200000:1.0:1787826243.368958:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000410e3391 507: 1< 0 - - > 2< 2076672 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.368960:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000dcaaf340 00000080:00200000:1.0:1787826243.368961:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 252. 00000080:00200000:1.0:1787826243.368962:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 508 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.370145:0:64727:0:(rw26.c:875:ll_write_end()) pos 2080768, len 4096, copied 4096 00000080:00200000:1.0:1787826243.370156:0:64727:0:(rw26.c:933:ll_write_end()) page@0000000093e0b27e[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 508] 00000080:00200000:1.0:1787826243.370159:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000129acb4b 17ffffc0001001 3:0 ffff907d92bf43a0 508 lru 00000080:00200000:1.0:1787826243.370165:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@000000002e158e83 508: 1< 0 - - > 2< 2080768 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.370167:0:64727:0:(rw26.c:933:ll_write_end()) end page@0000000093e0b27e 00000080:00200000:1.0:1787826243.370168:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 253. 00000080:00200000:1.0:1787826243.370171:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 509 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.370199:0:64727:0:(rw26.c:875:ll_write_end()) pos 2084864, len 4096, copied 4096 00000080:00200000:1.0:1787826243.370204:0:64727:0:(rw26.c:933:ll_write_end()) page@0000000014664dac[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 509] 00000080:00200000:1.0:1787826243.370206:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @00000000c8b7df7f 17ffffc0001001 3:0 ffff907d92bf4488 509 lru 00000080:00200000:1.0:1787826243.370211:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@00000000063a6685 509: 1< 0 - - > 2< 2084864 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.370213:0:64727:0:(rw26.c:933:ll_write_end()) end page@0000000014664dac 00000080:00200000:1.0:1787826243.370214:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 254. 00000080:00200000:1.0:1787826243.370217:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 510 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.370231:0:64727:0:(rw26.c:875:ll_write_end()) pos 2088960, len 4096, copied 4096 00000080:00200000:1.0:1787826243.370236:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000ea7396f3[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 510] 00000080:00200000:1.0:1787826243.370238:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @0000000046790f88 17ffffc0001001 3:0 ffff907d92bf4740 510 lru 00000080:00200000:1.0:1787826243.370244:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@000000006e7328f9 510: 1< 0 - - > 2< 2088960 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.370246:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000ea7396f3 00000080:00200000:1.0:1787826243.370247:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 255. 00000080:00200000:1.0:1787826243.370249:0:64727:0:(rw26.c:681:ll_write_begin()) Writing 511 of 0 to 4096 bytes 00000080:00200000:1.0:1787826243.370266:0:64727:0:(rw26.c:875:ll_write_end()) pos 2093056, len 4096, copied 4096 00000080:00200000:1.0:1787826243.370270:0:64727:0:(rw26.c:933:ll_write_end()) page@00000000e220adb4[3 000000000ba29125 2 1 000000002f5fabcc 65536 0 511] 00000080:00200000:1.0:1787826243.370273:0:64727:0:(rw26.c:933:ll_write_end()) vmpage @000000001d0f3b00 17ffffc0001001 3:0 ffff907da55bf2b8 511 lru 00000080:00200000:1.0:1787826243.370279:0:64727:0:(rw26.c:933:ll_write_end()) osc-page@000000009221bcb9 511: 1< 0 - - > 2< 2093056 0 0 0x0 0x0 | 000000005f5e6cd2 0000000075551c4a > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1787826243.370281:0:64727:0:(rw26.c:933:ll_write_end()) end page@00000000e220adb4 00000080:00200000:1.0:1787826243.370282:0:64727:0:(rw26.c:933:ll_write_end()) queued cl_page: 256. 00000080:00200000:1.0:1787826243.370286:0:64727:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 256, from 0, to 4096 prio 0 00000008:00010000:1.0:1787826243.370348:0:64727:0:(osc_cache.c:858:osc_extent_find()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.370371:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.370381:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.370389:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.370397:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.370412:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.370420:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.370428:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.370436:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.370446:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.370453:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.370462:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371321:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371333:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371343:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1787826243.371360:0:64727:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93313c38, count 15, dirtied 15 00000008:00010000:1.0:1787826243.371369:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371378:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371396:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371404:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371413:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371422:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371430:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371445:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371454:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371463:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371483:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371493:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371512:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371522:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371531:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1787826243.371544:0:64727:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93313c38, count 15, dirtied 15 00000008:00010000:1.0:1787826243.371551:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371560:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371569:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371577:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371586:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371595:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371604:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371630:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371639:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371655:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371664:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371673:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371693:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371702:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371710:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1787826243.371724:0:64727:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93313c38, count 15, dirtied 15 00000008:00010000:1.0:1787826243.371730:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371739:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371754:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371773:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371782:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371791:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371800:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371808:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371817:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371825:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371834:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371842:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371851:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371860:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371868:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1787826243.371881:0:64727:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93313c38, count 15, dirtied 15 00000008:00010000:1.0:1787826243.371887:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371896:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371904:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371912:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371924:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371931:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371938:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371945:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371953:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371960:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371968:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371988:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.371996:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372005:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372014:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1787826243.372027:0:64727:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93313c38, count 15, dirtied 15 00000008:00010000:1.0:1787826243.372032:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372040:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372049:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372056:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372065:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372074:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372087:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372096:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372105:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372114:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372124:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372133:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372141:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372149:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372159:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1787826243.372174:0:64727:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93313c38, count 15, dirtied 15 00000008:00010000:1.0:1787826243.372181:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372190:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372198:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372207:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372216:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372224:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372233:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372241:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372255:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372263:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372272:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372280:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372289:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372297:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372306:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1787826243.372319:0:64727:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93313c38, count 15, dirtied 15 00000008:00010000:1.0:1787826243.372325:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372339:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372348:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372357:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372366:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372375:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372383:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372391:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372399:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372408:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372417:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372425:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372432:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372438:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372445:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1787826243.372457:0:64727:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93313c38, count 15, dirtied 15 00000008:00010000:1.0:1787826243.372461:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372469:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372477:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372490:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372497:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372505:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372512:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372521:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372530:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372538:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372546:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372553:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372561:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372568:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372577:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1787826243.372592:0:64727:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93313c38, count 15, dirtied 15 00000008:00010000:1.0:1787826243.372597:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372635:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372647:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372655:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.372662:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373242:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373264:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373272:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373280:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373288:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373297:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373304:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373313:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373321:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373331:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1787826243.373346:0:64727:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93313c38, count 15, dirtied 15 00000008:00010000:1.0:1787826243.373352:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373359:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373369:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373377:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373384:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373392:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373401:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373415:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373423:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373431:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373439:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373446:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373454:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373461:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373480:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1787826243.373492:0:64727:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93313c38, count 15, dirtied 15 00000008:00010000:1.0:1787826243.373506:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373515:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373524:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373532:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373539:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373548:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373556:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373564:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373573:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373586:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373595:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373603:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373754:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373831:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373840:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1787826243.373855:0:64727:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93313c38, count 15, dirtied 15 00000008:00010000:1.0:1787826243.373861:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373871:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373885:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373893:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373901:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373910:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373919:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373928:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373936:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373945:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373954:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.373964:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374045:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374054:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374063:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1787826243.374077:0:64727:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93313c38, count 15, dirtied 15 00000008:00010000:1.0:1787826243.374083:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374092:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374101:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374110:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374123:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374131:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374138:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374146:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374154:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374163:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374171:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374179:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374187:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374197:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374206:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1787826243.374218:0:64727:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93313c38, count 15, dirtied 15 00000008:00010000:1.0:1787826243.374223:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374230:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374238:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374245:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374252:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374259:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374271:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374278:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374286:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374295:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374302:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374310:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374318:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374326:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374336:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1787826243.374350:0:64727:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93313c38, count 15, dirtied 15 00000008:00010000:1.0:1787826243.374356:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374365:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374374:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374705:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374720:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374729:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374738:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374747:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374827:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374836:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374844:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374852:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374859:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374867:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374875:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1787826243.374889:0:64727:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93313c38, count 15, dirtied 15 00000008:00010000:1.0:1787826243.374895:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374907:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374915:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374922:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374931:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374941:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374949:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374955:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.374963:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.375123:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.375139:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.375147:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.375156:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.375165:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826243.375175:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1787826243.375189:0:64727:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93313c38, count 15, dirtied 15 00000008:00010000:1.0:1787826243.375195:0:64727:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1787826243.375203:0:64727:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93313c38, count 1, dirtied 1 00000080:00200000:1.0:1787826243.375208:0:64727:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 256 pages 1048576 bytes, tot: 1048576 00000080:00200000:1.0:1787826243.375213:0:64727:0:(vvp_io.c:1421:vvp_io_write_start()) f202.sanity-flr: write bytes 0, result: 1048576 00000020:00010000:1.0:1787826243.375229:0:64727:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b1308 (ffff907d884a7bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1787826243.375233:0:64727:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b1308 (ffff907d884a7bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826243.375237:0:64727:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322e30 (ffff907da5608bd8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1787826243.375245:0:64727:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.375256:0:64727:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:1.0:1787826243.375274:0:64727:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b1308 (ffff907d884a7bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826243.375277:0:64727:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322e30 (ffff907d884a7bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826243.375288:0:64727:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000080:00200000:1.0:1787826243.375295:0:64727:0:(file.c:2207:ll_file_io_generic()) Range unlock [256, 511] 00000080:00200000:1.0:1787826243.375304:0:64727:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f7:0x0] ignore/verify layout 0/0, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826243.375308:0:64727:0:(file.c:2240:ll_file_io_generic()) f202.sanity-flr: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:1.0:1787826243.375320:0:64727:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f7:0x0] updating i_size 2097152 i_blocks 0 00000080:00200000:1.0:1787826243.375325:0:64727:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:1.0:1787826243.375362:0:64727:0:(file.c:479:ll_file_release()) START file f202.sanity-flr:[0x200000404:0x1f7:0x0](ffff907d93313a88), flags 100001 00000002:00100000:1.0:1787826243.375428:0:64727:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d908a30c0 x1874670011373312/t8589942897(8589942897) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 768/848 e 0 to 0 dl 1787826261 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826243.375439:0:64727:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826243.375444:0:64727:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0780 x1874670011374976/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826243.375478:0:64727:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:64727:1874670011374976:192.168.203.104@tcp:35:dd.0 00000100:00100000:1.0:1787826243.375591:0:64727:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d854a1b40 going to sleep for 11 seconds 00000100:00100000:1.0:1787826243.381688:0:64727:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:64727:1874670011374976:192.168.203.104@tcp:35:dd.0 00000002:00100000:1.0:1787826243.381722:0:64727:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d908a30c0 x1874670011373312/t8589942897(8589942897) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 768/848 e 0 to 0 dl 1787826261 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826243.407484:0:64729:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826243.407511:0:64729:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826243.407572:0:64729:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 64729 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826243.407584:0:64729:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64729 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826243.407595:0:64729:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64729 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826243.407641:0:64729:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64729:1874670011375104:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826243.407863:0:64729:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d82fb9540 going to sleep for 13 seconds 00000100:00100000:1.0:1787826243.421366:0:64729:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64729:1874670011375104:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826243.421393:0:64729:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.421406:0:64729:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.421418:0:64729:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.421427:0:64729:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.421439:0:64729:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.421448:0:64729:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.421455:0:64729:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826243.421477:0:64729:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d835252c0 x1874670011375104/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/688 e 0 to 0 dl 1787826261 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826243.421491:0:64729:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.421504:0:64729:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1787826243.421510:0:64729:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826243.421538:0:64729:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:1.0:1787826243.421545:0:64729:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b539 00000080:00010000:1.0:1787826243.421549:0:64729:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826243.421552:0:64729:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b539 from it ffff9dd140e2bbf8 00010000:00010000:1.0:1787826243.421555:0:64729:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.421563:0:64729:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826243.421587:0:64729:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826243.421644:0:64729:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), file ffff907d852ac780, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1787826243.421652:0:64729:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=open 00000002:00010000:1.0:1787826243.421672:0:64729:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f202.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:1.0:1787826243.421742:0:64729:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b540 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64729 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.421750:0:64729:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b540 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64729 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826243.421771:0:64729:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826243.421778:0:64729:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0780 x1874670011375360/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826243.421786:0:64729:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b540 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64729 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826243.421809:0:64729:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64729:1874670011375360:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826243.421991:0:64729:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d854a1b40 going to sleep for 13 seconds 00000100:00100000:1.0:1787826243.429670:0:64729:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64729:1874670011375360:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826243.429703:0:64729:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b540 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64729 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.429716:0:64729:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b540 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64729 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.429722:0:64729:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b540 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64729 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.429729:0:64729:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b540 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64729 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.429738:0:64729:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b540 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64729 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.429745:0:64729:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0a000 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826243.429749:0:64729:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b540 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64729 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.429756:0:64729:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b540 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64729 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.429772:0:64729:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b540 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 64729 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.429780:0:64729:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826243.429786:0:64729:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b540 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64729 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.429792:0:64729:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b540 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64729 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826243.429817:0:64729:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908a0780 x1874670011375360/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/608 e 0 to 0 dl 1787826261 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826243.429831:0:64729:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826243.430115:0:64729:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826243.430138:0:64729:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826243.430153:0:64729:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826243.430168:0:64729:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826243.430175:0:64729:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826243.430182:0:64729:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826243.430187:0:64729:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b539 00000080:00010000:1.0:1787826243.430193:0:64729:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b539 from it ffff9dd140e2bbb0 00010000:00010000:1.0:1787826243.430197:0:64729:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826243.430206:0:64729:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826243.430222:0:64729:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:1.0:1787826243.430234:0:64729:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00000080:00200080:1.0:1787826243.430275:0:64729:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=c0086915 arg=f533d0 00000100:00100000:1.0:1787826243.430344:0:64729:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64729:1874670011375488:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826243.430441:0:64729:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d854a1b40 going to sleep for 13 seconds 00000100:00100000:1.0:1787826243.435490:0:64729:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64729:1874670011375488:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826243.436546:0:64729:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00000080:00200000:2.0:1787826243.468446:0:64733:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826243.468478:0:64733:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826243.468499:0:64733:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826243.468518:0:64733:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1787826243.468527:0:64733:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826243.468535:0:64733:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826243.468539:0:64733:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b539 00000080:00010000:2.0:1787826243.468546:0:64733:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b539 from it ffff9dd141e57bf8 00010000:00010000:2.0:1787826243.468549:0:64733:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826243.468559:0:64733:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1787826243.468581:0:64733:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826243.468596:0:64733:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), file ffff907d8500a180, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1787826243.468602:0:64733:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=open 00000002:00010000:2.0:1787826243.468725:0:64733:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f202.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:2.0:1787826243.468855:0:64733:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a000/0xe5f736940357b547 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64733 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826243.468863:0:64733:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a000/0xe5f736940357b547 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64733 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826243.468874:0:64733:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826243.468886:0:64733:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114ad00 x1874670011375744/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826243.468896:0:64733:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a000/0xe5f736940357b547 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64733 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826243.468917:0:64733:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64733:1874670011375744:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826243.469099:0:64733:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da0039600 going to sleep for 13 seconds 00000100:00100000:3.0:1787826243.475425:0:64733:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64733:1874670011375744:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826243.475450:0:64733:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a000/0xe5f736940357b547 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64733 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826243.475463:0:64733:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a000/0xe5f736940357b547 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64733 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826243.475480:0:64733:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a000/0xe5f736940357b547 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64733 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826243.475487:0:64733:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a000/0xe5f736940357b547 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64733 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826243.475494:0:64733:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a000/0xe5f736940357b547 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64733 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826243.475500:0:64733:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907da563a000 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826243.475505:0:64733:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a000/0xe5f736940357b547 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64733 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826243.475511:0:64733:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a000/0xe5f736940357b547 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64733 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826243.475518:0:64733:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a000/0xe5f736940357b547 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 64733 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826243.475524:0:64733:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826243.475531:0:64733:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a000/0xe5f736940357b547 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64733 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826243.475537:0:64733:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563a000/0xe5f736940357b547 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64733 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826243.475555:0:64733:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d9114ad00 x1874670011375744/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/608 e 0 to 0 dl 1787826261 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826243.475568:0:64733:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:3.0:1787826243.475754:0:64733:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826243.475774:0:64733:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826243.475789:0:64733:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826243.475800:0:64733:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1787826243.475807:0:64733:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826243.475812:0:64733:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826243.475816:0:64733:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b539 00000080:00010000:3.0:1787826243.475821:0:64733:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b539 from it ffff9dd141e57bb0 00010000:00010000:3.0:1787826243.475824:0:64733:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826243.475838:0:64733:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1787826243.475851:0:64733:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:3.0:1787826243.475861:0:64733:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00000080:00200080:3.0:1787826243.475891:0:64733:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=c0086915 arg=1a983d0 00000100:00100000:3.0:1787826243.475943:0:64733:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83526580 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64733:1874670011375872:192.168.203.104@tcp:34:lfs.0 00000100:00100000:3.0:1787826243.477950:0:64733:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d82fb9d80 going to sleep for 13 seconds 00000100:00100000:3.0:1787826243.488377:0:64733:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83526580 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64733:1874670011375872:192.168.203.104@tcp:34:lfs.0 00000080:00200008:3.0:1787826243.488554:0:64733:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00000080:00200000:1.0:1787826243.529459:0:64735:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826243.529490:0:64735:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826243.529806:0:64735:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826243.529822:0:64735:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1787826243.529832:0:64735:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826243.529840:0:64735:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826243.529844:0:64735:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b539 00000080:00010000:1.0:1787826243.529852:0:64735:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b539 from it ffff9dd140e2bbf8 00010000:00010000:1.0:1787826243.529856:0:64735:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826243.529865:0:64735:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1787826243.529882:0:64735:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826243.529898:0:64735:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), file ffff907d87186300, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1787826243.529905:0:64735:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=open 00000002:00010000:1.0:1787826243.529920:0:64735:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f202.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:1.0:1787826243.529975:0:64735:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b54e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64735 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.529983:0:64735:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b54e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64735 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826243.529994:0:64735:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826243.530006:0:64735:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0780 x1874670011376128/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826243.530015:0:64735:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b54e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64735 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826243.530034:0:64735:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64735:1874670011376128:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826243.531473:0:64735:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d854a1b40 going to sleep for 13 seconds 00000100:00100000:1.0:1787826243.535917:0:64735:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64735:1874670011376128:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826243.535940:0:64735:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b54e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64735 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.535953:0:64735:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b54e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64735 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.535960:0:64735:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b54e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64735 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.535968:0:64735:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b54e lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64735 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.535977:0:64735:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b54e lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64735 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.535983:0:64735:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0a000 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826243.535988:0:64735:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b54e lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64735 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.536000:0:64735:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b54e lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64735 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.536008:0:64735:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b54e lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 64735 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.536016:0:64735:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826243.536024:0:64735:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b54e lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64735 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826243.536030:0:64735:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b54e lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64735 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826243.536048:0:64735:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908a0780 x1874670011376128/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/608 e 0 to 0 dl 1787826261 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826243.536062:0:64735:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826243.536362:0:64735:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826243.536379:0:64735:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826243.536393:0:64735:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826243.536405:0:64735:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1787826243.536411:0:64735:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826243.536417:0:64735:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826243.536420:0:64735:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b539 00000080:00010000:1.0:1787826243.536425:0:64735:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b539 from it ffff9dd140e2bbb0 00010000:00010000:1.0:1787826243.536429:0:64735:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826243.536435:0:64735:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1787826243.536447:0:64735:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:1.0:1787826243.536456:0:64735:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00000080:00200080:1.0:1787826243.536546:0:64735:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=c0086915 arg=e963d0 00000100:00100000:1.0:1787826243.536604:0:64735:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64735:1874670011376256:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826243.536871:0:64735:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d854a1b40 going to sleep for 13 seconds 00000100:00100000:1.0:1787826243.540211:0:64735:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:64735:1874670011376256:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826243.541121:0:64735:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00010000:00010000:2.0:1787826244.469170:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826246.520690:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826249.596396:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1787826250.404822:0:65292:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-flr test 203: mirror file preserve mirror ID === 06:24:08 (1787826248) 00000080:00200000:2.0:1787826250.590153:0:65308:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826250.590181:0:65308:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826250.590200:0:65308:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1787826250.590221:0:65308:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1787826250.590230:0:65308:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826250.590252:0:65308:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826250.590256:0:65308:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b539 00000080:00010000:2.0:1787826250.590263:0:65308:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b539 from it ffff9dd140c8fb58 00010000:00010000:2.0:1787826250.590267:0:65308:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1787826250.590277:0:65308:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1787826250.590292:0:65308:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826250.590305:0:65308:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), flags=13 00000080:00200000:2.0:1787826250.590310:0:65308:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=getattr 00000002:00010000:2.0:1787826250.590322:0:65308:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f203.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1787826250.590365:0:65308:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b555 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.590374:0:65308:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b555 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.590384:0:65308:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b555 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826250.590410:0:65308:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65308:1874670011376896:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826250.590814:0:65308:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88867900 going to sleep for 13 seconds 00000100:00100000:2.0:1787826250.611845:0:65308:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589942901 gen 1 00000100:00100000:2.0:1787826250.611959:0:65308:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65308:1874670011376896:192.168.203.104@tcp:101:lfs.0 00010000:00010000:2.0:1787826250.611983:0:65308:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b555 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.612001:0:65308:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b555 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.612007:0:65308:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b555 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.612015:0:65308:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b555 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.612025:0:65308:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b555 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.612032:0:65308:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907da563ac00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:2.0:1787826250.612037:0:65308:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b555 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.612044:0:65308:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b555 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.612052:0:65308:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b555 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.612059:0:65308:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1787826250.612068:0:65308:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b555 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.612076:0:65308:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b555 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826250.612099:0:65308:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff907d91148b40 x1874670011376896/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 584/584 e 0 to 0 dl 1787826268 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1787826250.612116:0:65308:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1787826250.612151:0:65308:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1787826250.612166:0:65308:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00010000:2.0:1787826250.612176:0:65308:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b539 from it ffff9dd140c8f930 00010000:00010000:2.0:1787826250.612179:0:65308:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1787826250.612188:0:65308:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1787826250.612796:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1787826250.616925:0:65308:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826250.616964:0:65308:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826250.617000:0:65308:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1787826250.617018:0:65308:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1787826250.617027:0:65308:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826250.617035:0:65308:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826250.617039:0:65308:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b539 00000080:00010000:2.0:1787826250.617047:0:65308:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b539 from it ffff9dd140c8fbe8 00010000:00010000:2.0:1787826250.617052:0:65308:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1787826250.617062:0:65308:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1787826250.617075:0:65308:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826250.617084:0:65308:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f203.sanity-flr, flags=16392 00000080:00200008:2.0:1787826250.617120:0:65308:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1787826250.617125:0:65308:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826250.617132:0:65308:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826250.617138:0:65308:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1787826250.617147:0:65308:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1787826250.617154:0:65308:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826250.617157:0:65308:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826250.617160:0:65308:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b539 00000080:00010000:2.0:1787826250.617164:0:65308:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b539 from it ffff9dd140c8fcb8 00010000:00010000:2.0:1787826250.617168:0:65308:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1787826250.617176:0:65308:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200008:2.0:1787826250.617810:0:65308:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1787826250.617816:0:65308:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826250.617825:0:65308:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826250.617835:0:65308:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1787826250.617844:0:65308:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:2.0:1787826250.617851:0:65308:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826250.617855:0:65308:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826250.617857:0:65308:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b539 00000080:00010000:2.0:1787826250.617861:0:65308:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b539 from it ffff9dd140c8fcb8 00010000:00010000:2.0:1787826250.617865:0:65308:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1787826250.617873:0:65308:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:2.0:1787826250.617993:0:65308:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826250.618000:0:65308:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826250.618007:0:65308:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1787826250.618016:0:65308:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:2.0:1787826250.618027:0:65308:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826250.618031:0:65308:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826250.618033:0:65308:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b539 00000080:00010000:2.0:1787826250.618037:0:65308:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b539 from it ffff9dd140c8fb58 00010000:00010000:2.0:1787826250.618040:0:65308:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1787826250.618047:0:65308:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:2.0:1787826250.618055:0:65308:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826250.618060:0:65308:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f203.sanity-flr, flags=8 00000080:00200000:2.0:1787826250.618074:0:65308:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826250.618080:0:65308:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826250.618085:0:65308:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1787826250.618092:0:65308:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:2.0:1787826250.618098:0:65308:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826250.618102:0:65308:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826250.618104:0:65308:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b539 00000080:00010000:2.0:1787826250.618107:0:65308:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b539 from it ffff9dd140c8fbf8 00010000:00010000:2.0:1787826250.618110:0:65308:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1787826250.618122:0:65308:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:2.0:1787826250.618130:0:65308:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826250.618135:0:65308:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f203.sanity-flr, flags=777 00000080:00200000:2.0:1787826250.618139:0:65308:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826250.618143:0:65308:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826250.618148:0:65308:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1787826250.618156:0:65308:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:2.0:1787826250.618162:0:65308:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826250.618165:0:65308:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826250.618167:0:65308:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b539 00000080:00010000:2.0:1787826250.618170:0:65308:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b539 from it ffff9dd140c8fb38 00010000:00010000:2.0:1787826250.618173:0:65308:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1787826250.618179:0:65308:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:2.0:1787826250.618185:0:65308:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 3 00000080:00200000:2.0:1787826250.618191:0:65308:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), file ffff907d934f3800, open_flags a141, mode 81b6 opened 0 00000080:00200000:2.0:1787826250.618201:0:65308:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=open|creat 00000002:00010000:2.0:1787826250.618223:0:65308:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f203.sanity-flr,[0x200000404:0x1f9:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0120502 00010000:00010000:2.0:1787826250.618238:0:65308:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1787826250.618297:0:65308:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1787826250.618304:0:65308:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1787826250.618307:0:65308:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86043e00/0xe5f736940357b539 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0f264 expref: -99 pid: 64729 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1787826250.618348:0:65308:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b55c lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.618354:0:65308:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b55c lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826250.618367:0:65308:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826250.618382:0:65308:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d91148b40 x1874670011377152/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 528/67896 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826250.618393:0:65308:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b55c lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826250.618419:0:65308:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65308:1874670011377152:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826250.618869:0:65308:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88867900 going to sleep for 13 seconds 00000100:00100000:2.0:1787826250.628516:0:65308:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65308:1874670011377152:192.168.203.104@tcp:101:lfs.0 00010000:00010000:2.0:1787826250.628551:0:65308:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b55c lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f313 expref: -99 pid: 65308 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.628585:0:65308:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b55c lrc: 4/0,1 mode: --/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f313 expref: -99 pid: 65308 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.628592:0:65308:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b55c lrc: 4/0,1 mode: --/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f313 expref: -99 pid: 65308 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.628602:0:65308:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b55c lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f313 expref: -99 pid: 65308 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.628659:0:65308:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b55c lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f313 expref: -99 pid: 65308 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.628674:0:65308:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b55c lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f313 expref: -99 pid: 65308 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.628683:0:65308:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b55c lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f313 expref: -99 pid: 65308 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.628690:0:65308:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b55c lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f313 expref: -99 pid: 65308 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826250.628704:0:65308:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x8000037, status=0 req@ffff907d91148b40 x1874670011377152/t8589942902(8589942902) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 528/608 e 0 to 0 dl 1787826268 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1787826250.628718:0:65308:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d91148b40 x1874670011377152/t8589942902(8589942902) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 528/608 e 0 to 0 dl 1787826268 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1787826250.628731:0:65308:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b55c lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f313 expref: -99 pid: 65308 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.628744:0:65308:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/505 (0 0) 00000080:00200000:2.0:1787826250.628751:0:65308:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1787826250.628766:0:65308:0:(namei.c:1628:ll_create_it()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=open|creat 00000080:00200000:2.0:1787826250.628804:0:65308:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988) 00000080:00200000:2.0:1787826250.628812:0:65308:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x1f9:0x0], updating i_size 0 00000080:00200000:2.0:1787826250.628850:0:65308:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1f9:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:2.0:1787826250.628854:0:65308:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) 00000080:00010000:2.0:1787826250.628858:0:65308:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) for lock 0xe5f736940357b55c 00000080:00010000:2.0:1787826250.628869:0:65308:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) 00000080:00200008:2.0:1787826250.628885:0:65308:0:(file.c:1090:ll_file_open()) START file f203.sanity-flr:[0x200000404:0x1f9:0x0](ffff907d84541988), flags 120501 00000002:00100000:2.0:1787826250.628904:0:65308:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d91148b40 x1874670011377152/t8589942902(8589942902) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 528/608 e 0 to 0 dl 1787826268 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1787826250.628947:0:65308:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b55c from it ffff907da005f3c0 00010000:00010000:2.0:1787826250.628951:0:65308:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b55c lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f313 expref: -99 pid: 65308 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.628962:0:65308:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b55c lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f313 expref: -99 pid: 65308 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826250.628989:0:65308:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988), inode mode 81a4 mask 40 00000080:00200000:2.0:1787826250.629066:0:65308:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988), inode mode 81a4 mask 2 00000080:00200000:2.0:1787826250.629078:0:65308:0:(xattr.c:414:ll_xattr_set()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988), xattr lov 00000002:00010000:2.0:1787826250.629099:0:65308:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f9:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 040000002 00010000:00010000:2.0:1787826250.629155:0:65308:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b563 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.629166:0:65308:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b563 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826250.629177:0:65308:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826250.629184:0:65308:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114bc00 x1874670011377408/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826250.629194:0:65308:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b563 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826250.629214:0:65308:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65308:1874670011377408:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826250.629434:0:65308:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88867180 going to sleep for 13 seconds 00000100:00100000:3.0:1787826250.632315:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826250.632456:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025476480 00000100:00100000:2.0:1787826250.632467:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025476480 00000100:00100000:2.0:1787826250.632894:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7111 00000100:00100000:2.0:1787826250.632908:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d835261c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32457:x1874670025476480:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826250.632933:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b55c lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0f313 expref: -99 pid: 65308 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826250.633028:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826250.633035:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d835261c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32457:x1874670025476480:12345-192.168.203.104@tcp:104: Request processed in 132us (735us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826250.633043:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7111 00000100:00100000:2.0:1787826250.638750:0:65308:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65308:1874670011377408:192.168.203.104@tcp:101:lfs.0 00010000:00010000:2.0:1787826250.638779:0:65308:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b563 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.638799:0:65308:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b563 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.638808:0:65308:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b563 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.638815:0:65308:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b563 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.638825:0:65308:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b563 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.638831:0:65308:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907da563ac00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:2.0:1787826250.638836:0:65308:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b563 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.638843:0:65308:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b563 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.638851:0:65308:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b563 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.638858:0:65308:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1787826250.638866:0:65308:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b563 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826250.638873:0:65308:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563ac00/0xe5f736940357b563 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 65308 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826250.638902:0:65308:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x800002b, status=0 req@ffff907d9114bc00 x1874670011377408/t8589942903(8589942903) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/848 e 0 to 0 dl 1787826268 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1787826250.638922:0:65308:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d9114bc00 x1874670011377408/t8589942903(8589942903) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/848 e 0 to 0 dl 1787826268 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1787826250.638940:0:65308:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1787826250.638994:0:65308:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f9:0x0] ignore/verify layout 1/0, layout version 1 restore needed 0 00000080:00200000:2.0:1787826250.639015:0:65308:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826250.639064:0:65308:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x1f9:0x0], updating i_size 0 00000002:00100000:2.0:1787826250.639080:0:65308:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d9114bc00 x1874670011377408/t8589942903(8589942903) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 760/848 e 0 to 0 dl 1787826268 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1787826250.639123:0:65308:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d9114bc00 x1874670011377408/t8589942903(8589942903) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 760/848 e 0 to 0 dl 1787826268 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1787826250.639135:0:65308:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1787826250.639141:0:65308:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114a580 x1874670011377792/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1787826250.639165:0:65308:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65308:1874670011377792:192.168.203.104@tcp:35:lfs.0 00000100:00100000:2.0:1787826250.639278:0:65308:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88867180 going to sleep for 11 seconds 00000100:00100000:2.0:1787826250.643513:0:65308:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65308:1874670011377792:192.168.203.104@tcp:35:lfs.0 00000002:00100000:2.0:1787826250.643555:0:65308:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d9114bc00 x1874670011377408/t8589942903(8589942903) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 760/848 e 0 to 0 dl 1787826268 ref 2 fl Complete:RQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1787826250.643668:0:65308:0:(file.c:479:ll_file_release()) START file f203.sanity-flr:[0x200000404:0x1f9:0x0](ffff907d84541988), flags 100001 00000002:00100000:2.0:1787826250.643747:0:65308:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d91148b40 x1874670011377152/t8589942902(8589942902) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 528/608 e 0 to 0 dl 1787826268 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1787826250.643761:0:65308:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1787826250.643767:0:65308:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114a940 x1874670011378048/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1787826250.643793:0:65308:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65308:1874670011378048:192.168.203.104@tcp:35:lfs.0 00000100:00100000:2.0:1787826250.643895:0:65308:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88867900 going to sleep for 11 seconds 00000100:00100000:2.0:1787826250.652078:0:65308:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65308:1874670011378048:192.168.203.104@tcp:35:lfs.0 00000002:00100000:2.0:1787826250.652125:0:65308:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d91148b40 x1874670011377152/t8589942902(8589942902) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 528/608 e 0 to 0 dl 1787826268 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826250.667992:0:65309:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826250.668647:0:65309:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826250.668741:0:65309:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042600/0xe5f736940357b56a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 65309 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826250.668755:0:65309:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042600/0xe5f736940357b56a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65309 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826250.668767:0:65309:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042600/0xe5f736940357b56a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65309 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826250.668800:0:65309:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65309:1874670011378176:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826250.669008:0:65309:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d82fb9900 going to sleep for 13 seconds 00000100:00100000:1.0:1787826250.676708:0:65309:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65309:1874670011378176:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826250.676747:0:65309:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042600/0xe5f736940357b56a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f33d expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.676762:0:65309:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042600/0xe5f736940357b56a lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f33d expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.676776:0:65309:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042600/0xe5f736940357b56a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f33d expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.676789:0:65309:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042600/0xe5f736940357b56a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f33d expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.676810:0:65309:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042600/0xe5f736940357b56a lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f33d expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.676818:0:65309:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042600/0xe5f736940357b56a lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f33d expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.676826:0:65309:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042600/0xe5f736940357b56a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f33d expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826250.676839:0:65309:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d83525680 x1874670011378176/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/688 e 0 to 0 dl 1787826268 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826250.676854:0:65309:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042600/0xe5f736940357b56a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f33d expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.676868:0:65309:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1787826250.676876:0:65309:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826250.676916:0:65309:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:1.0:1787826250.677020:0:65309:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b56a 00000080:00010000:1.0:1787826250.677029:0:65309:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826250.677033:0:65309:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b56a from it ffff9dd140c97bf8 00010000:00010000:1.0:1787826250.677037:0:65309:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042600/0xe5f736940357b56a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f33d expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.677047:0:65309:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042600/0xe5f736940357b56a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f33d expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826250.677160:0:65309:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826250.677187:0:65309:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), file ffff907d85152000, open_flags d900, mode 0 opened 0 00000080:00200000:1.0:1787826250.677195:0:65309:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=open 00000002:00010000:1.0:1787826250.677222:0:65309:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f203.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0154401 00010000:00010000:1.0:1787826250.677346:0:65309:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b571 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65309 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.677355:0:65309:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b571 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65309 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826250.677551:0:65309:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826250.677561:0:65309:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a21c0 x1874670011378432/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826250.677573:0:65309:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b571 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65309 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826250.677600:0:65309:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65309:1874670011378432:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826250.680846:0:65309:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d939b1c00 going to sleep for 13 seconds 00000100:00100000:1.0:1787826250.693733:0:65309:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65309:1874670011378432:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826250.693772:0:65309:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b571 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f359 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.693803:0:65309:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b571 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f359 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.693814:0:65309:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b571 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f359 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.693828:0:65309:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b571 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f359 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.693841:0:65309:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b571 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f359 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.693858:0:65309:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b571 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f359 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.693868:0:65309:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b571 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f359 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.693876:0:65309:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b571 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f359 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826250.693892:0:65309:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffff907d908a21c0 x1874670011378432/t8589942906(8589942906) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/848 e 0 to 0 dl 1787826268 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826250.693908:0:65309:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a21c0 x1874670011378432/t8589942906(8589942906) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/848 e 0 to 0 dl 1787826268 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826250.693933:0:65309:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 240 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b571 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f359 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826250.693950:0:65309:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b571 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f359 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.693963:0:65309:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/505 (0 0) 00000080:00200000:1.0:1787826250.693971:0:65309:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1787826250.694032:0:65309:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x1f9:0x0](ffff907d84541988): rc = 0 00000080:00200000:1.0:1787826250.694050:0:65309:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1f9:0x0]: layout version change: 4294967294 -> 2 00000080:00010000:1.0:1787826250.694056:0:65309:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) for lock 0xe5f736940357b571 00000080:00010000:1.0:1787826250.694070:0:65309:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) 00000080:00200008:1.0:1787826250.694203:0:65309:0:(file.c:1090:ll_file_open()) START file f203.sanity-flr:[0x200000404:0x1f9:0x0](ffff907d84541988), flags 154400 00000002:00100000:1.0:1787826250.694230:0:65309:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a21c0 x1874670011378432/t8589942906(8589942906) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 768/848 e 0 to 0 dl 1787826268 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1787826250.694288:0:65309:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b571 from it ffff907da56c2300 00010000:00010000:1.0:1787826250.694294:0:65309:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b571 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f359 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.694305:0:65309:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b571 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f359 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826250.694333:0:65309:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988), inode mode 81a4 mask 44 00000080:00200000:1.0:1787826250.694518:0:65309:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988), inode mode 81a4 mask 4 00000080:00200000:1.0:1787826250.694527:0:65309:0:(xattr.c:760:ll_xattr_get()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988), xattr lov 00000080:00200008:1.0:1787826250.694561:0:65309:0:(file.c:6313:ll_getattr_dentry()) START file f203.sanity-flr:[0x200000404:0x1f9:0x0](ffff907d84541988), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826250.694568:0:65309:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988),name=f203.sanity-flr 00000002:00010000:1.0:1787826250.694593:0:65309:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f9:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826250.694604:0:65309:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b571 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f359 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826250.694692:0:65309:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b571 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f359 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826250.694707:0:65309:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826250.694713:0:65309:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) 00000080:00010000:1.0:1787826250.694716:0:65309:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) for lock 0xe5f736940357b571 00000080:00010000:1.0:1787826250.694721:0:65309:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b571 from it ffff9dd140c97cd8 00010000:00010000:1.0:1787826250.694724:0:65309:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b571 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f359 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826250.694732:0:65309:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b571 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f359 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826250.694753:0:65309:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/1, layout version 3 restore needed 0 00020000:00200000:1.0:1787826250.694767:0:65309:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1f9:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826250.694772:0:65309:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826250.694777:0:65309:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1f9:0x0] 00000008:00010000:1.0:1787826250.694800:0:65309:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826250.694804:0:65309:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c322b0, osc lock ffff907d962f54b8, flags 0x400000400001002 00010000:00010000:1.0:1787826250.694860:0:65309:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b578 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x305:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65309 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.694873:0:65309:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b578 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x305:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65309 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.694884:0:65309:0:(ldlm_request.c:1139:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff907d908a3c00 x1874670011378560/t0(0) o101->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000020:00200000:1.0:1787826250.694936:0:65309:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:1.0:1787826250.694954:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1787826250.694975:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011378560:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826250.711318:0:2349:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b578 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x305:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65309 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.711356:0:2349:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b578 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x305:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f367 expref: -99 pid: 65309 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.711385:0:2349:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b578 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x305:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f367 expref: -99 pid: 65309 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.711399:0:2349:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b578 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x305:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f367 expref: -99 pid: 65309 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826250.711426:0:2349:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b578 lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x305:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f367 expref: -99 pid: 65309 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1787826250.711440:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1787826250.711466:0:2349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b578 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x305:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f367 expref: -99 pid: 65309 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.711477:0:2349:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b578 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x305:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f367 expref: -99 pid: 65309 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.711485:0:2349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b578 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x305:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f367 expref: -99 pid: 65309 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.711502:0:2349:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b578 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x305:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f367 expref: -99 pid: 65309 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1787826250.711530:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011378560:192.168.203.104@tcp:101:lfs.0 00000080:00200000:3.0:1787826250.712123:0:65309:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f9:0x0] updating i_size 0 i_blocks 0 00000020:00010000:3.0:1787826250.712133:0:65309:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5791b28 (ffff907d884a7bd8/0) at cl_lock_release():245 00000020:00010000:3.0:1787826250.712139:0:65309:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5791b28 (ffff907d884a7bd8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826250.712145:0:65309:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c322b0 (ffff907da5608bd8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826250.712150:0:65309:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5791b28 (ffff907d884a7bd8/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826250.712153:0:65309:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c322b0 (ffff907d884a7bd8/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826250.712173:0:65309:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/1, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:3.0:1787826250.712260:0:65309:0:(file.c:479:ll_file_release()) START file f203.sanity-flr:[0x200000404:0x1f9:0x0](ffff907d84541988), flags 154000 00000002:00100000:3.0:1787826250.712340:0:65309:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d908a21c0 x1874670011378432/t8589942906(8589942906) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 768/848 e 0 to 0 dl 1787826268 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1787826250.712353:0:65309:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1787826250.712359:0:65309:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83526940 x1874670011378816/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1787826250.712396:0:65309:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83526940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65309:1874670011378816:192.168.203.104@tcp:35:lfs.0 00000100:00100000:3.0:1787826250.712775:0:65309:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d82fb99c0 going to sleep for 11 seconds 00000100:00100000:1.0:1787826250.718314:0:65309:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83526940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65309:1874670011378816:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826250.718362:0:65309:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d908a21c0 x1874670011378432/t8589942906(8589942906) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 768/848 e 0 to 0 dl 1787826268 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826250.718873:0:65309:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826250.718899:0:65309:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826250.718934:0:65309:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042600/0xe5f736940357b56a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f33d expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826250.718953:0:65309:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042600/0xe5f736940357b56a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f33d expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826250.718962:0:65309:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826250.718969:0:65309:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826250.718974:0:65309:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b56a 00000080:00010000:1.0:1787826250.718985:0:65309:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b56a from it ffff9dd140c97bf8 00010000:00010000:1.0:1787826250.718990:0:65309:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042600/0xe5f736940357b56a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f33d expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826250.719000:0:65309:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042600/0xe5f736940357b56a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f33d expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826250.719022:0:65309:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826250.719031:0:65309:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f203.sanity-flr, flags=265 00000080:00200000:1.0:1787826250.719038:0:65309:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988), inode mode 81a4 mask 44 00000080:00200008:1.0:1787826250.719047:0:65309:0:(file.c:1090:ll_file_open()) START file f203.sanity-flr:[0x200000404:0x1f9:0x0](ffff907d84541988), flags 154400 00000002:00010000:1.0:1787826250.719065:0:65309:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f9:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240114401 00010000:00010000:1.0:1787826250.719375:0:65309:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b57f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65309 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.719385:0:65309:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b57f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65309 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826250.719399:0:65309:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826250.719409:0:65309:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a21c0 x1874670011379072/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826250.719725:0:65309:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b57f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65309 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826250.719757:0:65309:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65309:1874670011379072:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826250.720373:0:65309:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d939b1c00 going to sleep for 13 seconds 00000100:00100000:1.0:1787826250.744732:0:65309:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65309:1874670011379072:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826250.744785:0:65309:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b57f lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f375 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.744799:0:65309:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b57f lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f375 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.744820:0:65309:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b57f lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f375 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.744834:0:65309:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b57f lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f375 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826250.744851:0:65309:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x200002b, status=0 req@ffff907d908a21c0 x1874670011379072/t8589942908(8589942908) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/848 e 0 to 0 dl 1787826268 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826250.744866:0:65309:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a21c0 x1874670011379072/t8589942908(8589942908) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/848 e 0 to 0 dl 1787826268 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826250.744880:0:65309:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 240 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b57f lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f375 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826250.744900:0:65309:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b57f lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f375 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.744911:0:65309:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/505 (0 0) 00000080:00200000:1.0:1787826250.745527:0:65309:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826250.745662:0:65309:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1f9:0x0]: layout version change: 2 -> 2 00000080:00010000:1.0:1787826250.745669:0:65309:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) for lock 0xe5f736940357b57f 00000080:00010000:1.0:1787826250.745679:0:65309:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b57f from it ffff9dd140c97bf0 00010000:00010000:1.0:1787826250.745685:0:65309:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b57f lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f375 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.745698:0:65309:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b57f lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f375 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1787826250.745736:0:65309:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a21c0 x1874670011379072/t8589942908(8589942908) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 760/848 e 0 to 0 dl 1787826268 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826250.745877:0:65309:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988), inode mode 81a4 mask 4 00000080:00200000:1.0:1787826250.745887:0:65309:0:(xattr.c:760:ll_xattr_get()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988), xattr lov 00000080:00200008:1.0:1787826250.745915:0:65309:0:(file.c:6313:ll_getattr_dentry()) START file f203.sanity-flr:[0x200000404:0x1f9:0x0](ffff907d84541988), request_mask 2047, flags 0, foreign 0 00000100:00100000:3.0:1787826250.766668:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826250.766842:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025476608 00000100:00100000:2.0:1787826250.766855:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025476608 00000100:00100000:2.0:1787826250.766934:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7112 00000100:00100000:2.0:1787826250.766946:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83524780 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32456:x1874670025476608:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826250.766970:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b571 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0f359 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826250.767062:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826250.767069:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83524780 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32456:x1874670025476608:12345-192.168.203.104@tcp:104: Request processed in 125us (476us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826250.767078:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7112 00000002:00100000:1.0:1787826250.840359:0:65309:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d90844780 x1874670011380096/t8589942911(8589942911) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 768/848 e 0 to 0 dl 1787826268 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1787826250.840443:0:65309:0:(file.c:7076:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000404:0x1f9:0x0](ffff907d84541988) 00000002:00010000:1.0:1787826250.840467:0:65309:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f9:0x0]) in obj [0x200000404:0x1f9:0x0], intent: layout flags 0 00010000:00010000:1.0:1787826250.840541:0:65309:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b594 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 65309 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.840554:0:65309:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b594 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65309 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.840565:0:65309:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b594 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65309 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826250.840592:0:65309:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65309:1874670011380480:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826250.840772:0:65309:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d939b1c00 going to sleep for 13 seconds 00000080:00200000:1.0:1787826250.862907:0:65309:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000404:0x1fa:0x0](ffff907d9221db88) 00000080:00200000:1.0:1787826250.862916:0:65309:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x1fa:0x0], updating i_size 0 00000080:00200000:1.0:1787826250.862967:0:65309:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1fa:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:1.0:1787826250.862973:0:65309:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x200000404:0x1fa:0x0](ffff907d9221db88) 00000080:00010000:1.0:1787826250.862977:0:65309:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fa:0x0](ffff907d9221db88) for lock 0xe5f736940357b59b 00000080:00010000:1.0:1787826250.862989:0:65309:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1fa:0x0](ffff907d9221db88) 00000080:00200008:1.0:1787826250.863008:0:65309:0:(file.c:1090:ll_file_open()) START file . :VOLATILE:0000:7B1B0C93:fd=03:[0x200000404:0x1fa:0x0](ffff907d9221db88), flags 574702 00000002:00100000:1.0:1787826250.863032:0:65309:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90846940 x1874670011380736/t8589942913(8589942913) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 552/608 e 0 to 0 dl 1787826268 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1787826250.863080:0:65309:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b59b from it ffff907d832411e0 00010000:00010000:1.0:1787826250.863086:0:65309:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b59b lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f3c2 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.863097:0:65309:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b59b lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f3c2 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826250.863124:0:65309:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fa:0x0](ffff907d9221db88), inode mode 8180 mask 40 00000080:00200080:1.0:1787826250.863155:0:65309:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988) cmd=c00c66f3 arg=7ffca50eedc4 00000002:00010000:1.0:1787826250.863179:0:65309:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f9:0x0]) in obj [0x200000404:0x1f9:0x0], intent: open flags 01004040000002 00010000:00010000:1.0:1787826250.863362:0:65309:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a2 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65309 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.863372:0:65309:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x81000000001000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a2 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65309 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826250.863384:0:65309:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826250.863453:0:65309:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0b40 x1874670011380992/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826250.863464:0:65309:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a2 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x81000000000000 pid: 65309 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826250.863489:0:65309:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65309:1874670011380992:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826250.864129:0:65309:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d939b1cc0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826250.883046:0:65309:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65309:1874670011380992:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826250.883148:0:65309:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PW ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a2 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc0f3d7 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.883163:0:65309:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a2 lrc: 4/1,0 mode: --/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc0f3d7 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.883178:0:65309:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a2 lrc: 4/1,0 mode: PW/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0f3d7 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.883192:0:65309:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a2 lrc: 4/1,0 mode: PW/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc0f3d7 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.883268:0:65309:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a2 lrc: 5/1,1 mode: PW/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0f3d7 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.883278:0:65309:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a2 lrc: 5/1,1 mode: PW/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0f3d7 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.883285:0:65309:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a2 lrc: 4/0,1 mode: PW/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0f3d7 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826250.883299:0:65309:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x600002b, status=0 req@ffff907d908a0b40 x1874670011380992/t8589942914(8589942914) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/848 e 0 to 0 dl 1787826268 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826250.883314:0:65309:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a0b40 x1874670011380992/t8589942914(8589942914) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/848 e 0 to 0 dl 1787826268 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826250.883331:0:65309:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a2 lrc: 3/0,1 mode: PW/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc0f3d7 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.883350:0:65309:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 2 res 8589935620/505 (0 0) 00000080:00200000:1.0:1787826250.883359:0:65309:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1f9:0x0] suppgids 0 0: rc 0 00000002:00100000:1.0:1787826250.883420:0:65309:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a0b40 x1874670011380992/t8589942914(8589942914) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 760/848 e 0 to 0 dl 1787826268 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1787826250.883430:0:65309:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) for lock 0xe5f736940357b5a2 00000080:00010000:1.0:1787826250.883439:0:65309:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b5a2 from it ffff9dd140c97cc0 00010000:00010000:1.0:1787826250.883442:0:65309:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a2 lrc: 3/0,1 mode: PW/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0f3d7 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.883451:0:65309:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a2 lrc: 2/0,0 mode: PW/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0f3d7 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200080:1.0:1787826250.883529:0:65309:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988) cmd=c00c66f3 arg=9c0300 00010000:00010000:1.0:1787826250.883789:0:65309:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a2 lrc: 2/0,0 mode: PW/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81008000000000 nid: local remote: 0x7918213f4fc0f3d7 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.883806:0:65309:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a2 lrc: 0/0,0 mode: --/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x85809400000000 nid: local remote: 0x7918213f4fc0f3d7 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826250.883825:0:65309:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d908a0b40 x1874670011380992/t8589942914(8589942914) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 760/848 e 0 to 0 dl 1787826268 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826250.883836:0:65309:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826250.883843:0:65309:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0000 x1874670011381248/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 504/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826250.883871:0:65309:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65309:1874670011381248:192.168.203.104@tcp:35:lfs.0 00000100:00100000:1.0:1787826250.884004:0:65309:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d939b1cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1787826250.890137:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826250.890932:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025476736 00000100:00100000:3.0:1787826250.890948:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025476736 00000100:00100000:3.0:1787826250.891025:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7113 00000100:00100000:3.0:1787826250.891037:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91149680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32460:x1874670025476736:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826250.891061:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b594 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0f3b4 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826250.891155:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826250.891162:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91149680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32460:x1874670025476736:12345-192.168.203.104@tcp:104: Request processed in 128us (1042us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826250.891170:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7113 00000100:00100000:2.0:1787826250.912456:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826250.913150:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025476864 00000100:00100000:3.0:1787826250.913167:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025476864 00000100:00100000:3.0:1787826250.913262:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7114 00000100:00100000:3.0:1787826250.913278:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d9114b840 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32460:x1874670025476864:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826250.913305:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b59b lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0f3c2 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826250.913418:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826250.913426:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d9114b840 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32460:x1874670025476864:12345-192.168.203.104@tcp:104: Request processed in 152us (984us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826250.913435:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7114 00010000:00010000:1.0:1787826250.913645:0:60472:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826250.913657:0:60472:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b59b lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fa:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0f3c2 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826250.913667:0:60472:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883b200 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826250.913673:0:60472:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b59b lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fa:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc0f3c2 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826250.913684:0:60472:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b59b lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fa:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc0f3c2 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826250.913697:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b59b lrc: 3/0,0 mode: PR/PR res: [0x200000404:0x1fa:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x7918213f4fc0f3c2 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826250.913732:0:60472:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x1fa:0x0]: losing layout lock 00010000:00010000:1.0:1787826250.913789:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b59b lrc: 2/0,0 mode: --/PR res: [0x200000404:0x1fa:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0f3c2 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826250.913798:0:60472:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826250.913831:0:60472:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b59b lrc: 1/0,0 mode: --/PR res: [0x200000404:0x1fa:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0f3c2 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826250.913841:0:60472:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b59b lrc: 0/0,0 mode: --/PR res: [0x200000404:0x1fa:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0f3c2 expref: -99 pid: 65309 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826250.913866:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826250.913868:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826250.913870:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826250.913880:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1787826250.913898:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011381504:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826250.923545:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011381504:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826250.925080:0:65309:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65309:1874670011381248:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826250.925125:0:65309:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d908a0b40 x1874670011380992/t8589942914(8589942914) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 760/848 e 0 to 0 dl 1787826268 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1787826250.925173:0:65309:0:(file.c:7076:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000404:0x1f9:0x0](ffff907d84541988) 00000002:00010000:1.0:1787826250.925188:0:65309:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f9:0x0]) in obj [0x200000404:0x1f9:0x0], intent: layout flags 0 00010000:00010000:1.0:1787826250.925256:0:65309:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 65309 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.925268:0:65309:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65309 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.925280:0:65309:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65309 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826250.925303:0:65309:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65309:1874670011381632:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826250.925407:0:65309:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d939b1cc0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826250.935796:0:65309:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65309:1874670011381632:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826250.935843:0:65309:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f3f3 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.935862:0:65309:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f3f3 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826250.935885:0:65309:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x400 disposition=0x1, status=0 req@ffff907d908a0000 x1874670011381632/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/480 e 0 to 0 dl 1787826268 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826250.935897:0:65309:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 136 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f3f3 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826250.935911:0:65309:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f3f3 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.935926:0:65309:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/505 (0 0) 00000080:00200000:1.0:1787826250.935934:0:65309:0:(namei.c:952:ll_intent_lock()) intent lock 1024 on i1 [0x200000404:0x1f9:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1787826250.935954:0:65309:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) for lock 0xe5f736940357b5a9 00000080:00010000:1.0:1787826250.935964:0:65309:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b5a9 from it ffff9dd140c97bc8 00010000:00010000:1.0:1787826250.935969:0:65309:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f3f3 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.935979:0:65309:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f3f3 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826250.936006:0:65309:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f3f3 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826250.936019:0:65309:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f3f3 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:1.0:1787826250.936028:0:65309:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x1f9:0x0](00000000ce105185) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f3f3 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826250.936085:0:65309:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f3f3 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826250.936093:0:65309:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f3f3 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826250.936143:0:65309:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826250.936156:0:65309:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1f9:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826250.936160:0:65309:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826250.936167:0:65309:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1787826250.936215:0:65309:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1787826250.936229:0:65309:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f3f3 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826250.936238:0:65309:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f3f3 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:1.0:1787826250.936246:0:65309:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x1f9:0x0](00000000ce105185) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f3f3 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826250.936262:0:65309:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826250.936267:0:65309:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1f9:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826250.936269:0:65309:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826250.936271:0:65309:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1787826250.936299:0:65309:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1787826250.936316:0:65309:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 0 blocks 0 [cma]time 0 0 0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b578 lrc: 1/0,0 mode: PR/PR res: [0x280000400:0x305:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f367 expref: -99 pid: 65309 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826250.936393:0:65309:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1f9:0x0]: layout version change: 4294967294 -> 3 00010000:00010000:1.0:1787826250.936399:0:65309:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f3f3 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826250.936406:0:65309:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f3f3 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:1.0:1787826250.936447:0:65309:0:(file.c:479:ll_file_release()) START file . :VOLATILE:0000:7B1B0C93:fd=03:[0x200000404:0x1fa:0x0](ffff907d9221db88), flags 554002 00000002:00100000:1.0:1787826250.936502:0:65309:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d90846940 x1874670011380736/t8589942913(8589942913) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 552/608 e 0 to 0 dl 1787826268 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826250.936515:0:65309:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826250.936521:0:65309:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0000 x1874670011381888/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826250.936548:0:65309:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65309:1874670011381888:192.168.203.104@tcp:35:lfs.0 00000100:00100000:1.0:1787826250.938243:0:65309:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d939b1c00 going to sleep for 11 seconds 00000100:00100000:1.0:1787826250.951087:0:65309:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65309:1874670011381888:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826250.951138:0:65309:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d90846940 x1874670011380736/t8589942913(8589942913) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 552/608 e 0 to 0 dl 1787826268 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1787826250.951217:0:65309:0:(file.c:479:ll_file_release()) START file f203.sanity-flr:[0x200000404:0x1f9:0x0](ffff907d84541988), flags 154002 00000002:00100000:1.0:1787826250.951294:0:65309:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d90844b40 x1874670011379328/t8589942909(8589942909) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 760/848 e 0 to 0 dl 1787826268 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826250.951305:0:65309:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826250.951310:0:65309:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0b40 x1874670011382144/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826250.951348:0:65309:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65309:1874670011382144:192.168.203.104@tcp:35:lfs.0 00000100:00100000:1.0:1787826250.951467:0:65309:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d939b1c00 going to sleep for 11 seconds 00000100:00100000:1.0:1787826250.953820:0:65309:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65309:1874670011382144:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826250.953862:0:65309:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d90844b40 x1874670011379328/t8589942909(8589942909) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 760/848 e 0 to 0 dl 1787826268 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826250.970688:0:65310:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826250.970731:0:65310:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826250.970822:0:65310:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 65310 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.970837:0:65310:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65310 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.970850:0:65310:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65310 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826250.970884:0:65310:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65310:1874670011382272:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826250.974045:0:65310:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d939b1c00 going to sleep for 13 seconds 00000100:00100000:1.0:1787826250.977980:0:65310:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65310:1874670011382272:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826250.978017:0:65310:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.978033:0:65310:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.978048:0:65310:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.978067:0:65310:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.978082:0:65310:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.978091:0:65310:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.978100:0:65310:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826250.978114:0:65310:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d908a0b40 x1874670011382272/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/688 e 0 to 0 dl 1787826268 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826250.978131:0:65310:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.978148:0:65310:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1787826250.978157:0:65310:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826250.978192:0:65310:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:1.0:1787826250.978200:0:65310:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b5b0 00000080:00010000:1.0:1787826250.978207:0:65310:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826250.978211:0:65310:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b5b0 from it ffff9dd141c2fbf8 00010000:00010000:1.0:1787826250.978214:0:65310:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.978230:0:65310:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826250.978260:0:65310:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826250.978289:0:65310:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), file ffff907d84ea7380, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1787826250.978298:0:65310:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=open 00000002:00010000:1.0:1787826250.978333:0:65310:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f203.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:1.0:1787826250.978401:0:65310:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b5b7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65310 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826250.978409:0:65310:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b5b7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65310 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826250.978422:0:65310:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826250.978430:0:65310:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0b40 x1874670011382528/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826250.978441:0:65310:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b5b7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65310 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826250.978466:0:65310:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65310:1874670011382528:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826251.020969:0:65311:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65311:1874670011382912:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826251.021002:0:65311:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b5be lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65311 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.021019:0:65311:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b5be lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65311 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.021033:0:65311:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b5be lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65311 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.021041:0:65311:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b5be lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65311 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.021051:0:65311:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b5be lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65311 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.021059:0:65311:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883ba00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826251.021065:0:65311:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b5be lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 65311 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.021072:0:65311:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b5be lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 65311 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.021080:0:65311:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b5be lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 65311 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.021088:0:65311:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826251.021096:0:65311:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b5be lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65311 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.021103:0:65311:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b5be lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65311 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826251.021128:0:65311:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d90845a40 x1874670011382912/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/608 e 0 to 0 dl 1787826269 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826251.021146:0:65311:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826251.022500:0:65311:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826251.022537:0:65311:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826251.022559:0:65311:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826251.022576:0:65311:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1787826251.022584:0:65311:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826251.022593:0:65311:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826251.022596:0:65311:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b5b0 00000080:00010000:1.0:1787826251.022776:0:65311:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b5b0 from it ffff9dd141c0fbb0 00010000:00010000:1.0:1787826251.022784:0:65311:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826251.022794:0:65311:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1787826251.022812:0:65311:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:1.0:1787826251.022826:0:65311:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00000080:00200080:1.0:1787826251.022888:0:65311:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=c0086915 arg=19c13d0 00000100:00100000:1.0:1787826251.022975:0:65311:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65311:1874670011383040:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826251.023151:0:65311:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d85569a80 going to sleep for 13 seconds 00000100:00100000:1.0:1787826251.034099:0:65311:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65311:1874670011383040:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826251.034345:0:65311:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00000080:00200000:2.0:1787826251.064932:0:65312:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826251.064964:0:65312:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826251.064986:0:65312:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826251.065002:0:65312:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1787826251.065017:0:65312:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826251.065025:0:65312:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826251.065029:0:65312:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b5b0 00000080:00010000:2.0:1787826251.065038:0:65312:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b5b0 from it ffff9dd141c37bb0 00010000:00010000:2.0:1787826251.065043:0:65312:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826251.065054:0:65312:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1787826251.065072:0:65312:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:2.0:1787826251.065086:0:65312:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00000080:00200080:2.0:1787826251.065130:0:65312:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=c0086915 arg=20f53d0 00000100:00100000:2.0:1787826251.065873:0:65312:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65312:1874670011383424:192.168.203.104@tcp:34:lfs.0 00000100:00100000:2.0:1787826251.067086:0:65312:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88867180 going to sleep for 13 seconds 00000080:00200000:1.0:1787826251.085374:0:65313:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826251.085397:0:65313:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826251.085414:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826251.085429:0:65313:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1787826251.085436:0:65313:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826251.085442:0:65313:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826251.085446:0:65313:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b5b0 00000080:00010000:1.0:1787826251.085453:0:65313:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b5b0 from it ffff9dd141853bf8 00010000:00010000:1.0:1787826251.085456:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826251.085466:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1787826251.085480:0:65313:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826251.085493:0:65313:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), file ffff907d82cac300, open_flags 4d902, mode 0 opened 0 00000080:00200000:1.0:1787826251.085499:0:65313:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=open 00000002:00010000:1.0:1787826251.085510:0:65313:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f203.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 01154403 00010000:00010000:1.0:1787826251.085554:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.085561:0:65313:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826251.085572:0:65313:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826251.085583:0:65313:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a1a40 x1874670011383680/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826251.085598:0:65313:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826251.085654:0:65313:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011383680:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826251.085829:0:65313:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d85569a80 going to sleep for 13 seconds 00000100:00100000:1.0:1787826251.092714:0:65313:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011383680:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826251.092740:0:65313:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.092757:0:65313:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.092770:0:65313:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.092781:0:65313:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.092790:0:65313:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.092802:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.092810:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.092823:0:65313:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826251.092833:0:65313:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffff907d908a1a40 x1874670011383680/t8589942921(8589942921) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/744 e 0 to 0 dl 1787826269 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826251.092845:0:65313:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a1a40 x1874670011383680/t8589942921(8589942921) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/744 e 0 to 0 dl 1787826269 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826251.092858:0:65313:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 136 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826251.092872:0:65313:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.092881:0:65313:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/505 (0 0) 00000080:00200000:1.0:1787826251.092887:0:65313:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1787826251.092930:0:65313:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x1f9:0x0](ffff907d84541988): rc = 0 00000080:00200000:1.0:1787826251.092940:0:65313:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1f9:0x0]: layout version change: 3 -> 3 00000080:00010000:1.0:1787826251.092945:0:65313:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) for lock 0xe5f736940357b5cc 00000080:00010000:1.0:1787826251.092955:0:65313:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) 00000080:00200008:1.0:1787826251.092969:0:65313:0:(file.c:1090:ll_file_open()) START file f203.sanity-flr:[0x200000404:0x1f9:0x0](ffff907d84541988), flags 1154402 00000002:00100000:1.0:1787826251.092986:0:65313:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a1a40 x1874670011383680/t8589942921(8589942921) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 664/744 e 0 to 0 dl 1787826269 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1787826251.093032:0:65313:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b5cc from it ffff907da56c28a0 00010000:00010000:1.0:1787826251.093037:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.093049:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826251.093069:0:65313:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988), inode mode 81a4 mask 46 00000080:00200080:1.0:1787826251.093100:0:65313:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988) cmd=800466af arg=7fffab60f6fc 00000080:00200000:1.0:1787826251.093197:0:65313:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826251.093210:0:65313:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826251.093220:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826251.093230:0:65313:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1787826251.093237:0:65313:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826251.093241:0:65313:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826251.093244:0:65313:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b5b0 00000080:00010000:1.0:1787826251.093249:0:65313:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b5b0 from it ffff9dd141853bf8 00010000:00010000:1.0:1787826251.093253:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826251.093260:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1787826251.093268:0:65313:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826251.093281:0:65313:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826251.093286:0:65313:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826251.093293:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1787826251.093300:0:65313:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1787826251.093307:0:65313:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826251.093311:0:65313:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826251.093314:0:65313:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b5b0 00000080:00010000:1.0:1787826251.093317:0:65313:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b5b0 from it ffff9dd141853b38 00010000:00010000:1.0:1787826251.093320:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1787826251.093327:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1787826251.093334:0:65313:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 3 00000080:00200000:1.0:1787826251.093338:0:65313:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=. :VOLATILE:0000:1B88D17B:fd=03, dir=[0x200000007:0x1:0x0](ffff907d84619148), file ffff907d85561800, open_flags 2f9c1, mode 8180 opened 0 00000080:00200000:1.0:1787826251.093346:0:65313:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=. :VOLATILE:0000:1B88D17B:fd=03, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=open|creat 00000002:00010000:1.0:1787826251.093363:0:65313:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: . :VOLATILE:0000:1B88D17B:fd=03,[0x200000404:0x1fb:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0574702 00010000:00010000:1.0:1787826251.093373:0:65313:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1787826251.093413:0:65313:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1787826251.093419:0:65313:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826251.093423:0:65313:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b5b0 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0f408 expref: -99 pid: 65310 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1787826251.093455:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b5d3 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.093461:0:65313:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b5d3 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826251.093471:0:65313:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826251.094638:0:65313:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0b40 x1874670011383936/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 552/67896 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826251.094654:0:65313:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b5d3 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826251.094680:0:65313:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011383936:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826251.094833:0:65313:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d855693c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826251.099505:0:65313:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011383936:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826251.099543:0:65313:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b5d3 lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4b7 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.099578:0:65313:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b5d3 lrc: 4/0,1 mode: --/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4b7 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.099587:0:65313:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b5d3 lrc: 4/0,1 mode: --/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4b7 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.099601:0:65313:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b5d3 lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f4b7 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.099669:0:65313:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b5d3 lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4b7 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.099685:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b5d3 lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f4b7 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.099694:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b5d3 lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f4b7 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.099702:0:65313:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b5d3 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f4b7 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826251.099716:0:65313:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x8000037, status=0 req@ffff907d908a0b40 x1874670011383936/t8589942922(8589942922) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 552/608 e 0 to 0 dl 1787826269 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826251.099730:0:65313:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a0b40 x1874670011383936/t8589942922(8589942922) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 552/608 e 0 to 0 dl 1787826269 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826251.099744:0:65313:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b5d3 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4b7 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.099756:0:65313:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/507 (0 0) 00000080:00200000:1.0:1787826251.099763:0:65313:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1787826251.099779:0:65313:0:(namei.c:1628:ll_create_it()) VFS Op:name=. :VOLATILE:0000:1B88D17B:fd=03, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=open|creat 00000080:00200000:1.0:1787826251.099824:0:65313:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000404:0x1fb:0x0](ffff907d9221cb08) 00000080:00200000:1.0:1787826251.099832:0:65313:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x1fb:0x0], updating i_size 0 00000080:00200000:1.0:1787826251.099875:0:65313:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1fb:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:1.0:1787826251.099880:0:65313:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x200000404:0x1fb:0x0](ffff907d9221cb08) 00000080:00010000:1.0:1787826251.099883:0:65313:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fb:0x0](ffff907d9221cb08) for lock 0xe5f736940357b5d3 00000080:00010000:1.0:1787826251.100076:0:65313:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1fb:0x0](ffff907d9221cb08) 00000080:00200008:1.0:1787826251.100095:0:65313:0:(file.c:1090:ll_file_open()) START file . :VOLATILE:0000:1B88D17B:fd=03:[0x200000404:0x1fb:0x0](ffff907d9221cb08), flags 574701 00000002:00100000:1.0:1787826251.100118:0:65313:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a0b40 x1874670011383936/t8589942922(8589942922) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 552/608 e 0 to 0 dl 1787826269 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1787826251.100164:0:65313:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b5d3 from it ffff907da56c28a0 00010000:00010000:1.0:1787826251.100169:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b5d3 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f4b7 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.100180:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b5d3 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f4b7 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826251.100326:0:65313:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fb:0x0](ffff907d9221cb08), inode mode 8180 mask 40 00000080:00200000:1.0:1787826251.100467:0:65313:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fb:0x0](ffff907d9221cb08), inode mode 8180 mask 2 00000080:00200000:1.0:1787826251.100482:0:65313:0:(xattr.c:414:ll_xattr_set()) VFS Op:inode=[0x200000404:0x1fb:0x0](ffff907d9221cb08), xattr lov 00000002:00010000:1.0:1787826251.100505:0:65313:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1fb:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 040000002 00010000:00010000:1.0:1787826251.100574:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b5da lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.100582:0:65313:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b5da lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826251.100595:0:65313:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826251.100603:0:65313:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a3c00 x1874670011384192/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826251.100647:0:65313:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b5da lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826251.100673:0:65313:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011384192:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826251.101225:0:65313:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d855693c0 going to sleep for 13 seconds 00000100:00100000:3.0:1787826251.106038:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826251.106460:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025477120 00000100:00100000:2.0:1787826251.106472:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025477120 00000100:00100000:2.0:1787826251.106541:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7115 00000100:00100000:2.0:1787826251.106553:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83526940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32456:x1874670025477120:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826251.106579:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b5d3 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0f4b7 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826251.106726:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826251.106734:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83526940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32456:x1874670025477120:12345-192.168.203.104@tcp:104: Request processed in 184us (711us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826251.106747:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7115 00000100:00100000:1.0:1787826251.113514:0:65313:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011384192:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826251.113545:0:65313:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b5da lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.113562:0:65313:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b5da lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.113570:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b5da lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.113578:0:65313:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b5da lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.113588:0:65313:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b5da lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.113596:0:65313:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0a800 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826251.113601:0:65313:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b5da lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.113654:0:65313:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b5da lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.113662:0:65313:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b5da lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.113670:0:65313:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826251.113680:0:65313:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b5da lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.113688:0:65313:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b5da lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826251.113720:0:65313:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x800002b, status=0 req@ffff907d908a3c00 x1874670011384192/t8589942923(8589942923) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/768 e 0 to 0 dl 1787826269 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826251.113748:0:65313:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a3c00 x1874670011384192/t8589942923(8589942923) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/768 e 0 to 0 dl 1787826269 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826251.113768:0:65313:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826251.113827:0:65313:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fb:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1787826251.113838:0:65313:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fb:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826251.113912:0:65313:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x1fb:0x0], updating i_size 0 00000002:00100000:1.0:1787826251.113932:0:65313:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a3c00 x1874670011384192/t8589942923(8589942923) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 680/768 e 0 to 0 dl 1787826269 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826251.113979:0:65313:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d908a3c00 x1874670011384192/t8589942923(8589942923) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 680/768 e 0 to 0 dl 1787826269 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826251.113989:0:65313:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826251.113995:0:65313:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a21c0 x1874670011384576/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826251.114021:0:65313:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011384576:192.168.203.104@tcp:35:lfs.0 00000100:00100000:1.0:1787826251.114116:0:65313:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d85569300 going to sleep for 11 seconds 00000100:00100000:1.0:1787826251.121339:0:65313:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011384576:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826251.121385:0:65313:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d908a3c00 x1874670011384192/t8589942923(8589942923) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 680/768 e 0 to 0 dl 1787826269 ref 2 fl Complete:RQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1787826251.121466:0:65313:0:(file.c:6313:ll_getattr_dentry()) START file f203.sanity-flr:[0x200000404:0x1f9:0x0](ffff907d84541988), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826251.121477:0:65313:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988),name=f203.sanity-flr 00000002:00010000:1.0:1787826251.121506:0:65313:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f9:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826251.121532:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826251.121550:0:65313:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826251.121560:0:65313:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826251.121567:0:65313:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) 00000080:00010000:1.0:1787826251.121572:0:65313:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) for lock 0xe5f736940357b5cc 00000080:00010000:1.0:1787826251.121581:0:65313:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b5cc from it ffff9dd141853cd8 00010000:00010000:1.0:1787826251.121585:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826251.121596:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826251.121678:0:65313:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1787826251.121694:0:65313:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1f9:0x0] 00000008:00010000:1.0:1787826251.121720:0:65313:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826251.121839:0:65313:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c32e30, osc lock ffff907d962f4d68, flags 0x400001002 00010000:00010000:1.0:1787826251.121906:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.121918:0:65313:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1787826251.122124:0:65313:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826251.128822:0:2349:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.128862:0:2349:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.128882:0:2349:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.128896:0:2349:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826251.128917:0:2349:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1787826251.128933:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1787826251.128953:0:2349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.128962:0:2349:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.128970:0:2349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.128981:0:2349:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1787826251.129006:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011384704:192.168.203.104@tcp:101:lfs.0 00000080:00200000:3.0:1787826251.129102:0:65313:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f9:0x0] updating i_size 0 i_blocks 0 00000020:00010000:3.0:1787826251.129110:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98aff8c8 (ffff907da5608e18/0) at cl_lock_release():245 00000020:00010000:3.0:1787826251.129115:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98aff8c8 (ffff907da5608e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826251.129121:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c32e30 (ffff907da5608638/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826251.129126:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98aff8c8 (ffff907da5608e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826251.129129:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c32e30 (ffff907da5608e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826251.129146:0:65313:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:3.0:1787826251.129178:0:65313:0:(file.c:6313:ll_getattr_dentry()) START file . :VOLATILE:0000:1B88D17B:fd=03:[0x200000404:0x1fb:0x0](ffff907d9221cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787826251.129189:0:65313:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1fb:0x0](ffff907d9221cb08),name=. :VOLATILE:0000:1B88D17B:fd=03 00000002:00010000:3.0:1787826251.129224:0:65313:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1fb:0x0]) in obj [0x200000404:0x1fb:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826251.129298:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042a00/0xe5f736940357b5e8 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fb:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.129309:0:65313:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042a00/0xe5f736940357b5e8 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fb:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.129325:0:65313:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042a00/0xe5f736940357b5e8 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fb:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826251.129358:0:65313:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011384832:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826251.129479:0:65313:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d82fb99c0 going to sleep for 13 seconds 00000100:00100000:3.0:1787826251.138564:0:65313:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011384832:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826251.138604:0:65313:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042a00/0xe5f736940357b5e8 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fb:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4e8 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.138643:0:65313:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042a00/0xe5f736940357b5e8 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4e8 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.138658:0:65313:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042a00/0xe5f736940357b5e8 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f4e8 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.138672:0:65313:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042a00/0xe5f736940357b5e8 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4e8 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.138689:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042a00/0xe5f736940357b5e8 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f4e8 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.138698:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042a00/0xe5f736940357b5e8 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f4e8 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.138705:0:65313:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042a00/0xe5f736940357b5e8 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f4e8 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826251.138725:0:65313:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d83525680 x1874670011384832/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/744 e 0 to 0 dl 1787826269 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1787826251.138738:0:65313:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 160 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042a00/0xe5f736940357b5e8 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4e8 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1787826251.139111:0:65313:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042a00/0xe5f736940357b5e8 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4e8 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826251.139131:0:65313:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/507 (0 0) 00000080:00200000:2.0:1787826251.139140:0:65313:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1fb:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1787826251.139432:0:65313:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1fb:0x0]: layout version change: 4294967294 -> 1 00000080:00010000:2.0:1787826251.139440:0:65313:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fb:0x0](ffff907d9221cb08) for lock 0xe5f736940357b5e8 00000080:00010000:2.0:1787826251.139448:0:65313:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1fb:0x0](ffff907d9221cb08) 00000080:00010000:2.0:1787826251.139452:0:65313:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b5e8 from it ffff9dd141853cd8 00010000:00010000:2.0:1787826251.139456:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042a00/0xe5f736940357b5e8 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f4e8 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826251.139467:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042a00/0xe5f736940357b5e8 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f4e8 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826251.139515:0:65313:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fb:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1787826251.139528:0:65313:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1fb:0x0] 00000008:00010000:2.0:1787826251.139552:0:65313:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826251.139556:0:65313:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88867db0, osc lock ffff907d830a3380, flags 0x400001002 00000008:00010000:2.0:1787826251.139566:0:65313:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826251.139568:0:65313:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88867df0, osc lock ffff907d830a2c30, flags 0x400001002 00010000:00010000:2.0:1787826251.139681:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008905cba5/0xe5f736940357b5ef lrc: 3/1,0 mode: --/PR res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826251.139692:0:65313:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008905cba5/0xe5f736940357b5ef lrc: 3/1,0 mode: --/PR res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826251.139758:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1787826251.139782:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011384960:192.168.203.104@tcp:101:lfs.0 00010000:00010000:2.0:1787826251.140837:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000051aef7d5/0xe5f736940357b5f6 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826251.140858:0:65313:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000051aef7d5/0xe5f736940357b5f6 lrc: 3/1,0 mode: --/PR res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1787826251.140901:0:65313:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000100:00100000:3.0:1787826251.141352:0:2351:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1787826251.141376:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011385088:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826251.144693:0:2350:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008905cba5/0xe5f736940357b5ef lrc: 6/2,0 mode: --/PR res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.144729:0:2350:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008905cba5/0xe5f736940357b5ef lrc: 6/2,0 mode: --/PR res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4ef expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.144747:0:2350:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008905cba5/0xe5f736940357b5ef lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f4ef expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.144760:0:2350:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008905cba5/0xe5f736940357b5ef lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4ef expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:3.0:1787826251.144779:0:2350:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008905cba5/0xe5f736940357b5ef lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f4ef expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:3.0:1787826251.144793:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826251.144798:0:2350:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008905cba5/0xe5f736940357b5ef lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f4ef expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.144809:0:2350:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008905cba5/0xe5f736940357b5ef lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f4ef expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.144816:0:2350:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008905cba5/0xe5f736940357b5ef lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f4ef expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.144825:0:2350:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008905cba5/0xe5f736940357b5ef lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4ef expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1787826251.144849:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011384960:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826251.149330:0:2351:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000051aef7d5/0xe5f736940357b5f6 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.149376:0:2351:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000051aef7d5/0xe5f736940357b5f6 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4f6 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.149394:0:2351:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000051aef7d5/0xe5f736940357b5f6 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f4f6 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.149406:0:2351:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000051aef7d5/0xe5f736940357b5f6 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f4f6 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:3.0:1787826251.149434:0:2351:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000051aef7d5/0xe5f736940357b5f6 lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f4f6 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:3.0:1787826251.149449:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:3.0:1787826251.149466:0:2351:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000051aef7d5/0xe5f736940357b5f6 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f4f6 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.149476:0:2351:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000051aef7d5/0xe5f736940357b5f6 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f4f6 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.149486:0:2351:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000051aef7d5/0xe5f736940357b5f6 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f4f6 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.149496:0:2351:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000051aef7d5/0xe5f736940357b5f6 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4f6 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1787826251.149523:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011385088:192.168.203.104@tcp:101:lfs.0 00000080:00200000:2.0:1787826251.149665:0:65313:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fb:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1787826251.149673:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98afe018 (ffff907da56087e8/0) at cl_lock_release():245 00000020:00010000:2.0:1787826251.149677:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98afe018 (ffff907da56087e8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826251.149683:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88867db0 (ffff907da5608cf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826251.149688:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88867df0 (ffff907d884a7cf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826251.149691:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98afe018 (ffff907da56087e8/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826251.149694:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88867db0 (ffff907da56087e8/1) at cl_lock_fini():72 00000020:00010000:2.0:1787826251.149699:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88867df0 (ffff907da56087e8/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826251.149715:0:65313:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fb:0x0] ignore/verify layout 0/1, layout version 1 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:2.0:1787826251.149747:0:65313:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988) cmd=c00c66f3 arg=7fffab611844 00000002:00010000:2.0:1787826251.149779:0:65313:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f9:0x0]) in obj [0x200000404:0x1f9:0x0], intent: open flags 01004040000001 00010000:00010000:2.0:1787826251.149851:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b5fd lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826251.149860:0:65313:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x81000000001000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b5fd lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826251.149877:0:65313:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826251.149898:0:65313:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d911483c0 x1874670011385344/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826251.149910:0:65313:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b5fd lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x81000000000000 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826251.149936:0:65313:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011385344:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826251.150206:0:65313:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88867d80 going to sleep for 13 seconds 00000100:00100000:1.0:1787826251.154935:0:65313:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011385344:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826251.154974:0:65313:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PW ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b5fd lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc0f504 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.154990:0:65313:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b5fd lrc: 4/1,0 mode: --/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc0f504 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.155006:0:65313:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b5fd lrc: 4/1,0 mode: PW/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0f504 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.155020:0:65313:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b5fd lrc: 4/1,0 mode: PW/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc0f504 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.155037:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b5fd lrc: 5/1,1 mode: PW/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0f504 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.155047:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b5fd lrc: 5/1,1 mode: PW/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0f504 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.155055:0:65313:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b5fd lrc: 4/0,1 mode: PW/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0f504 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826251.155070:0:65313:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x600002b, status=0 req@ffff907d911483c0 x1874670011385344/t8589942925(8589942925) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/744 e 0 to 0 dl 1787826269 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826251.155085:0:65313:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d911483c0 x1874670011385344/t8589942925(8589942925) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/744 e 0 to 0 dl 1787826269 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826251.155103:0:65313:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b5fd lrc: 3/0,1 mode: PW/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc0f504 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.155118:0:65313:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 2 res 8589935620/505 (0 0) 00000080:00200000:1.0:1787826251.155125:0:65313:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1f9:0x0] suppgids 0 0: rc 0 00000002:00100000:1.0:1787826251.155147:0:65313:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d911483c0 x1874670011385344/t8589942925(8589942925) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 656/744 e 0 to 0 dl 1787826269 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1787826251.155157:0:65313:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) for lock 0xe5f736940357b5fd 00000080:00010000:1.0:1787826251.155165:0:65313:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b5fd from it ffff9dd141853cc0 00010000:00010000:1.0:1787826251.155169:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b5fd lrc: 3/0,1 mode: PW/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0f504 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.155178:0:65313:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b5fd lrc: 2/0,0 mode: PW/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0f504 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200008:1.0:1787826251.155280:0:65313:0:(file.c:6313:ll_getattr_dentry()) START file f203.sanity-flr:[0x200000404:0x1f9:0x0](ffff907d84541988), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826251.155290:0:65313:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988),name=f203.sanity-flr 00000002:00010000:1.0:1787826251.155318:0:65313:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f9:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826251.155331:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826251.155362:0:65313:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826251.155371:0:65313:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826251.155379:0:65313:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) 00000080:00010000:1.0:1787826251.155384:0:65313:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) for lock 0xe5f736940357b5cc 00000080:00010000:1.0:1787826251.155389:0:65313:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b5cc from it ffff9dd141853cd8 00010000:00010000:1.0:1787826251.155392:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826251.155402:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1787826251.155443:0:65313:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1787826251.155459:0:65313:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1f9:0x0] 00000008:00010000:1.0:1787826251.155483:0:65313:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826251.155487:0:65313:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c32b30, osc lock ffff907d962f5248, flags 0x400001002 00010000:00010000:1.0:1787826251.155510:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826251.155523:0:65313:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1787826251.155539:0:65313:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826251.155544:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826251.155559:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1787826251.155570:0:65313:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826251.155584:0:65313:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f9:0x0] updating i_size 0 i_blocks 0 00000020:00010000:1.0:1787826251.155589:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98afe780 (ffff907da5608f38/0) at cl_lock_release():245 00000020:00010000:1.0:1787826251.155593:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98afe780 (ffff907da5608f38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826251.155596:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c32b30 (ffff907da5608128/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826251.155602:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98afe780 (ffff907da5608f38/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826251.155648:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c32b30 (ffff907da5608f38/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826251.155664:0:65313:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826251.155678:0:65313:0:(file.c:6313:ll_getattr_dentry()) START file f203.sanity-flr:[0x200000404:0x1f9:0x0](ffff907d84541988), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826251.155682:0:65313:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988),name=f203.sanity-flr 00000002:00010000:1.0:1787826251.155691:0:65313:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f9:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826251.155698:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826251.155708:0:65313:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1787826251.155716:0:65313:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826251.155720:0:65313:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) 00000080:00010000:1.0:1787826251.155723:0:65313:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) for lock 0xe5f736940357b5cc 00000080:00010000:1.0:1787826251.155727:0:65313:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b5cc from it ffff9dd141853cd8 00010000:00010000:1.0:1787826251.155730:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826251.155738:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1787826251.155751:0:65313:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/1, layout version 3 restore needed 0 00000001:00010000:1.0:1787826251.155756:0:65313:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1f9:0x0] 00000008:00010000:1.0:1787826251.155763:0:65313:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826251.155765:0:65313:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c32b30, osc lock ffff907d962f5248, flags 0x400001002 00010000:00010000:1.0:1787826251.155772:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826251.155782:0:65313:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1787826251.155791:0:65313:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826251.155795:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826251.155803:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1787826251.155812:0:65313:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826251.155816:0:65313:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f9:0x0] updating i_size 0 i_blocks 0 00000020:00010000:1.0:1787826251.155819:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98afe780 (ffff907da5608f38/0) at cl_lock_release():245 00000020:00010000:1.0:1787826251.155822:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98afe780 (ffff907da5608f38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826251.155825:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c32b30 (ffff907da5608128/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826251.155828:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98afe780 (ffff907da5608f38/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826251.155831:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c32b30 (ffff907da5608f38/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826251.155838:0:65313:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:1.0:1787826251.155849:0:65313:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988) cmd=801066da arg=7fffab611740 00000080:00200000:1.0:1787826251.155860:0:65313:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/0, layout version 3 restore needed 0 00020000:00200000:1.0:1787826251.155868:0:65313:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:1.0:1787826251.155969:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1787826251.155991:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011385472:192.168.203.104@tcp:1:lfs.0 00000100:00100000:1.0:1787826251.159760:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011385472:192.168.203.104@tcp:1:lfs.0 00000080:00200000:2.0:1787826251.160840:0:65313:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988), inode mode 81a4 mask 4 00000080:00200000:2.0:1787826251.160859:0:65313:0:(xattr.c:760:ll_xattr_get()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988), xattr lov 00000080:00200008:2.0:1787826251.161275:0:65313:0:(file.c:6313:ll_getattr_dentry()) START file f203.sanity-flr:[0x200000404:0x1f9:0x0](ffff907d84541988), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1787826251.161288:0:65313:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988),name=f203.sanity-flr 00000002:00010000:2.0:1787826251.161324:0:65313:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f9:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826251.161479:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826251.161500:0:65313:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1787826251.161510:0:65313:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826251.161520:0:65313:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) 00000080:00010000:2.0:1787826251.161524:0:65313:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) for lock 0xe5f736940357b5cc 00000080:00010000:2.0:1787826251.161534:0:65313:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b5cc from it ffff9dd141853cd8 00010000:00010000:2.0:1787826251.161538:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826251.161554:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1787826251.161589:0:65313:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/1, layout version 3 restore needed 0 00000001:00010000:2.0:1787826251.161604:0:65313:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1f9:0x0] 00000008:00010000:2.0:1787826251.161797:0:65313:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826251.161802:0:65313:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322430, osc lock ffff907d830a2c30, flags 0x400001002 00010000:00010000:2.0:1787826251.161825:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826251.161838:0:65313:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:2.0:1787826251.161852:0:65313:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826251.161856:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826251.161864:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:2.0:1787826251.161876:0:65313:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826251.161964:0:65313:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f9:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1787826251.161971:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98afe780 (ffff907da5608f38/0) at cl_lock_release():245 00000020:00010000:2.0:1787826251.161975:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98afe780 (ffff907da5608f38/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826251.161978:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322430 (ffff907da5608128/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826251.161985:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98afe780 (ffff907da5608f38/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826251.161988:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322430 (ffff907da5608f38/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826251.162001:0:65313:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826251.207864:0:65313:0:(file.c:5224:ll_file_seek()) START file f203.sanity-flr:[0x200000404:0x1f9:0x0], offset: 0, type: (null) 00000080:00200000:2.0:1787826251.207889:0:65313:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/1, layout version 3 restore needed 0 00000001:00010000:2.0:1787826251.207903:0:65313:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1f9:0x0] 00000008:00010000:2.0:1787826251.207930:0:65313:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826251.207935:0:65313:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322430, osc lock ffff907d830a2c30, flags 0x400001002 00010000:00010000:2.0:1787826251.207967:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826251.207990:0:65313:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:2.0:1787826251.208101:0:65313:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826251.208109:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826251.208120:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:2.0:1787826251.208139:0:65313:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826251.208151:0:65313:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f9:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1787826251.208157:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98afe780 (ffff907da5608f38/0) at cl_lock_release():245 00000020:00010000:2.0:1787826251.208161:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98afe780 (ffff907da5608f38/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826251.208165:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322430 (ffff907da5608128/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826251.208170:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98afe780 (ffff907da5608f38/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826251.208173:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322430 (ffff907da5608f38/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826251.208192:0:65313:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/1, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826251.208201:0:65313:0:(file.c:5262:ll_file_seek()) COMPLETED file f203.sanity-flr:[0x200000404:0x1f9:0x0], offset: 0, type: (null), rc = 0 00000080:00200008:2.0:1787826251.208207:0:65313:0:(file.c:5224:ll_file_seek()) START file f203.sanity-flr:[0x200000404:0x1f9:0x0], offset: 0, type: SEEK_HOLE 00000080:00200000:2.0:1787826251.208218:0:65313:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/0, layout version 3 restore needed 0 00020000:00200000:2.0:1787826251.208226:0:65313:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1787826251.208246:0:65313:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f9:0x0] updating i_size 0 i_blocks 0 00000080:00200000:2.0:1787826251.208256:0:65313:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826251.208263:0:65313:0:(file.c:1848:ll_io_set_mirror()) f203.sanity-flr: desiginated mirror: 0 00000080:00200000:2.0:1787826251.208265:0:65313:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1787826251.208269:0:65313:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1787826251.208274:0:65313:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 18446744073709551615] 00000008:00010000:2.0:1787826251.208281:0:65313:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:2.0:1787826251.208283:0:65313:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322430, osc lock ffff907d830a2c30, flags 0x0 00010000:00010000:2.0:1787826251.208292:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826251.208302:0:65313:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826251.208311:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826251.208319:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826251.208326:0:65313:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1787826251.208342:0:65313:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f9:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1787826251.208348:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d800248a8 (ffff907da5608f38/0) at cl_lock_release():245 00000020:00010000:2.0:1787826251.208350:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d800248a8 (ffff907da5608f38/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826251.208353:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322430 (ffff907da5608128/1) at cl_lock_cancel():142 00010000:00010000:2.0:1787826251.208357:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826251.208366:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000020:00010000:2.0:1787826251.208375:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d800248a8 (ffff907da5608f38/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826251.208378:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322430 (ffff907da5608f38/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826251.208384:0:65313:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/0, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826251.208387:0:65313:0:(file.c:5262:ll_file_seek()) COMPLETED file f203.sanity-flr:[0x200000404:0x1f9:0x0], offset: 0, type: SEEK_HOLE, rc = -6 00000080:00200080:2.0:1787826251.208436:0:65313:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988) cmd=66f4 arg=67c 00000080:00200008:2.0:1787826251.208457:0:65313:0:(file.c:2485:do_file_read_iter()) START file f203.sanity-flr:[0x200000404:0x1f9:0x0], ppos: 0, count: 67108864 00000080:00200000:2.0:1787826251.208468:0:65313:0:(file.c:2090:ll_file_io_generic()) f203.sanity-flr: read ppos: 0, bytes: 67108864 00000080:00200000:2.0:1787826251.208477:0:65313:0:(file.c:1848:ll_io_set_mirror()) f203.sanity-flr: desiginated mirror: 0 00000020:00200000:2.0:1787826251.208483:0:65313:0:(cl_io.c:218:cl_io_rw_init()) header@000000000c895bc8[0x110, 2, [0x200000404:0x1f9:0x0] hash] 00000020:00200000:2.0:1787826251.208486:0:65313:0:(cl_io.c:218:cl_io_rw_init()) io range: 1 [0, 67108864) 2048 0 00000080:00200000:2.0:1787826251.208488:0:65313:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/0, layout version 3 restore needed 0 00000080:00200000:2.0:1787826251.208498:0:65313:0:(file.c:2170:ll_file_io_generic()) Range lock [0, 16383] 00020000:00200000:2.0:1787826251.208506:0:65313:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 67108864) 67108864, 67108864 00020000:00200000:2.0:1787826251.208508:0:65313:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1787826251.208518:0:65313:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 16383] 00000008:00010000:2.0:1787826251.208524:0:65313:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 11 00000008:00010000:2.0:1787826251.208528:0:65313:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322430, osc lock ffff907d830a2c30, flags 0x40000 00000080:00200000:2.0:1787826251.208533:0:65313:0:(vvp_io.c:874:vvp_io_read_start()) f203.sanity-flr: read [0, 67108864) 00000001:00010000:2.0:1787826251.208539:0:65313:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1f9:0x0] 00000008:00010000:2.0:1787826251.208543:0:65313:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826251.208545:0:65313:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322eb0, osc lock ffff907d830a3380, flags 0x400001002 00010000:00010000:2.0:1787826251.208553:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:2.0:1787826251.208564:0:65313:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000020:00200000:2.0:1787826251.208572:0:65313:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826251.208575:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:2.0:1787826251.208582:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000020:00200000:2.0:1787826251.208591:0:65313:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826251.208595:0:65313:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1f9:0x0] updating i_size 0 i_blocks 0 00000020:00010000:2.0:1787826251.208597:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98afe780 (ffff907da5608f38/0) at cl_lock_release():245 00000020:00010000:2.0:1787826251.208600:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98afe780 (ffff907da5608f38/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826251.208602:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322eb0 (ffff907da5608128/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826251.208664:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98afe780 (ffff907da5608f38/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826251.208668:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322eb0 (ffff907da5608f38/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826251.208674:0:65313:0:(vvp_io.c:932:vvp_io_read_start()) count:67108864 iocb pos:0 00000080:00200000:2.0:1787826251.208680:0:65313:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988), size=67108864 (max 335544320), offset=0=0x0, pages 16384 (max 81920), lockless, parallel 00000020:00010000:2.0:1787826251.208689:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d800248a8 (ffff907da5608f38/0) at cl_lock_release():245 00000020:00010000:2.0:1787826251.208692:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d800248a8 (ffff907da5608f38/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826251.208694:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322430 (ffff907da5608128/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826251.208697:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d800248a8 (ffff907da5608f38/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826251.208699:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322430 (ffff907da5608f38/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826251.208704:0:65313:0:(vvp_io.c:533:vvp_io_advance()) advancing 0 bytes 00000020:00200000:2.0:1787826251.208708:0:65313:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826251.208711:0:65313:0:(file.c:2207:ll_file_io_generic()) Range unlock [0, 16383] 00000080:00200000:2.0:1787826251.208716:0:65313:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/0, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826251.208720:0:65313:0:(file.c:2240:ll_file_io_generic()) f203.sanity-flr: 1 io complete with rc: 0, result: 0, restart: 0 00000020:00200000:2.0:1787826251.208722:0:65313:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826251.208726:0:65313:0:(file.c:2326:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:2.0:1787826251.208739:0:65313:0:(file.c:5401:ll_fsync()) START file: name=. :VOLATILE:0000:1B88D17B:fd=03, fid=[0x200000404:0x1fb:0x0], start=0, end=9223372036854775807, datasync=0, need_sync 1/0 00000100:00100000:2.0:1787826251.208806:0:65313:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011385600:192.168.203.104@tcp:44:lfs.0 00000100:00100000:2.0:1787826251.208939:0:65313:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d888673c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826251.217131:0:65313:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589942925 gen 1 00000100:00100000:1.0:1787826251.217260:0:65313:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d908a1a40 x1874670011383680/t8589942921(8589942921) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 664/744 e 0 to 0 dl 1787826269 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1787826251.217273:0:65313:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d908a0b40 x1874670011383936/t8589942922(8589942922) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 552/608 e 0 to 0 dl 1787826269 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1787826251.217322:0:65313:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d911483c0 x1874670011385344/t8589942925(8589942925) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 656/744 e 0 to 0 dl 1787826269 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1787826251.217335:0:65313:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011385600:192.168.203.104@tcp:44:lfs.0 00000080:00200000:1.0:1787826251.218145:0:65313:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fb:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826251.218165:0:65313:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:3.0:1787826251.219062:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826251.219108:0:40339:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:3.0:1787826251.219122:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025477376 00000100:00100000:3.0:1787826251.219129:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025477376 00000100:00100000:3.0:1787826251.219383:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7116 00000100:00100000:3.0:1787826251.219395:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d84e75050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:6934:x1874670025477376:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826251.219413:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000723b235b/0xe5f736940357b578 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x305:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc0f367 expref: -99 pid: 65309 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826251.219466:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826251.219472:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d84e75050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:6934:x1874670025477376:12345-192.168.203.104@tcp:104: Request processed in 81us (413us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826251.219479:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7116 00000080:00200000:1.0:1787826251.232365:0:65313:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fb:0x0] updating i_size 0 i_blocks 0 00000080:00200000:1.0:1787826251.232394:0:65313:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fb:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:1.0:1787826251.245169:0:65313:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988) cmd=801066da arg=7fffab611740 00000080:00200000:1.0:1787826251.245204:0:65313:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826251.245223:0:65313:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:1.0:1787826251.245450:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1787826251.245479:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011386112:192.168.203.104@tcp:1:lfs.0 00000080:00200008:2.0:1787826251.259890:0:65313:0:(llite_lib.c:2427:ll_setattr_raw()) START file . :VOLATILE:0000:1B88D17B:fd=03:[0x200000404:0x1fb:0x0](ffff907d9221cb08) current size 0, valid attrs 101f0, mode 907d, uid 0, gid 0, new size 4294967296, atime 1787826250.000000000, mtime 1787826250.000000000, ctime 1787826251.000000000 00010000:00010000:2.0:1787826251.259956:0:65313:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042a00/0xe5f736940357b5e8 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1a/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0f4e8 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826251.260174:0:65313:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x1fb:0x0]: losing layout lock 00010000:00010000:2.0:1787826251.260251:0:65313:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042a00/0xe5f736940357b5e8 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0f4e8 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826251.260260:0:65313:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1787826251.260264:0:65313:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042a00/0xe5f736940357b5e8 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x1fb:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0f4e8 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000020:00100000:2.0:1787826251.260302:0:65313:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1787826251.260315:0:65313:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114a1c0 x1874670011386368/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787826251.260397:0:65313:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011386368:192.168.203.104@tcp:36:lfs.0 00000100:00100000:2.0:1787826251.260591:0:65313:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d888670c0 going to sleep for 13 seconds 00000100:00100000:3.0:1787826251.287586:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826251.287880:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025477504 00000100:00100000:2.0:1787826251.287893:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025477504 00000100:00100000:3.0:1787826251.288048:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7117 00000100:00100000:3.0:1787826251.288062:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d835261c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:35506:x1874670025477504:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826251.288086:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008905cba5/0xe5f736940357b5ef lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x322:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0f4ef expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826251.288206:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826251.288214:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d835261c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:35506:x1874670025477504:12345-192.168.203.104@tcp:104: Request processed in 156us (642us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826251.288223:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7117 00000100:00100000:1.0:1787826251.289359:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011386752:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1787826251.296447:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826251.296547:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025477632 00000100:00100000:3.0:1787826251.296561:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025477632 00000100:00100000:3.0:1787826251.296668:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7118 00000100:00100000:3.0:1787826251.296679:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91148780 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:5933:x1874670025477632:12345-192.168.203.104@tcp:105: 00010000:00010000:3.0:1787826251.296700:0:40338:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1dd66eb/0xe5f736940357b60b lrc: 5/0,1 mode: --/PW res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0f535 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.296717:0:40338:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1dd66eb/0xe5f736940357b60b lrc: 5/0,1 mode: --/PW res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0f535 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.296729:0:40338:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1dd66eb/0xe5f736940357b60b lrc: 5/0,1 mode: --/PW res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc0f535 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.296757:0:40338:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1dd66eb/0xe5f736940357b60b lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10080000020000 nid: local remote: 0x7918213f4fc0f535 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.296770:0:40338:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1dd66eb/0xe5f736940357b60b lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10080000020000 nid: local remote: 0x7918213f4fc0f535 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.296782:0:40338:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1dd66eb/0xe5f736940357b60b lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80000020000 nid: local remote: 0x7918213f4fc0f535 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.296817:0:40338:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff907d8883a800) 00000100:00100000:3.0:1787826251.296898:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91148780 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:5933:x1874670025477632:12345-192.168.203.104@tcp:105: Request processed in 221us (467us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826251.296907:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7118 00000100:00100000:3.0:1787826251.304981:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:1.0:1787826251.305264:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011386752:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1787826251.306471:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025477760 00000100:00100000:2.0:1787826251.306484:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025477760 00000100:00100000:2.0:1787826251.306642:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7119 00000100:00100000:2.0:1787826251.306654:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83524780 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:28560:x1874670025477760:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826251.306680:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000051aef7d5/0xe5f736940357b5f6 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x306:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0f4f6 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826251.306811:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826251.306819:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83524780 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:28560:x1874670025477760:12345-192.168.203.104@tcp:104: Request processed in 167us (1854us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826251.306828:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7119 00010000:00010000:1.0:1787826251.306889:0:47852:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826251.306898:0:47852:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000051aef7d5/0xe5f736940357b5f6 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x306:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0f4f6 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826251.306907:0:47852:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907da563a600 already unused, calling callback (ffffffffc118e410) 00010000:00010000:1.0:1787826251.306914:0:47852:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000051aef7d5/0xe5f736940357b5f6 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x306:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x7918213f4fc0f4f6 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826251.306982:0:47852:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fb:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1787826251.307000:0:47852:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fb:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1787826251.307059:0:47852:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000051aef7d5/0xe5f736940357b5f6 lrc: 2/0,0 mode: --/PR res: [0x280000400:0x306:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0f4f6 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826251.307067:0:47852:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826251.307109:0:47852:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000051aef7d5/0xe5f736940357b5f6 lrc: 1/0,0 mode: --/PR res: [0x280000400:0x306:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0f4f6 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826251.307119:0:47852:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000051aef7d5/0xe5f736940357b5f6 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x306:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0f4f6 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826251.307138:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826251.307140:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826251.307141:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826251.307152:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1787826251.307168:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011387008:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1787826251.312424:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826251.312595:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025477888 00000100:00100000:2.0:1787826251.312635:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025477888 00000100:00100000:2.0:1787826251.312704:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7120 00000100:00100000:2.0:1787826251.312714:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83526940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:28562:x1874670025477888:12345-192.168.203.104@tcp:105: 00010000:00010000:2.0:1787826251.312757:0:40339:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001b7e234c/0xe5f736940357b612 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f53c expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826251.312773:0:40339:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001b7e234c/0xe5f736940357b612 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f53c expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826251.312790:0:40339:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001b7e234c/0xe5f736940357b612 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f53c expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826251.312803:0:40339:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001b7e234c/0xe5f736940357b612 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10080000000000 nid: local remote: 0x7918213f4fc0f53c expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826251.312815:0:40339:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001b7e234c/0xe5f736940357b612 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10080000000000 nid: local remote: 0x7918213f4fc0f53c expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826251.312828:0:40339:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001b7e234c/0xe5f736940357b612 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80000000000 nid: local remote: 0x7918213f4fc0f53c expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826251.312852:0:40339:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff907d8883b800) 00000100:00100000:2.0:1787826251.312929:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83526940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:28562:x1874670025477888:12345-192.168.203.104@tcp:105: Request processed in 216us (529us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826251.312938:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7120 00000100:00100000:1.0:1787826251.313228:0:65313:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011386880:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826251.313256:0:65313:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001b7e234c/0xe5f736940357b612 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0f53c expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.313276:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001b7e234c/0xe5f736940357b612 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc0f53c expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826251.313291:0:65313:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001b7e234c/0xe5f736940357b612 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000020000 nid: local remote: 0x7918213f4fc0f53c expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.313319:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001b7e234c/0xe5f736940357b612 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc0f53c expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.313328:0:65313:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001b7e234c/0xe5f736940357b612 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc0f53c expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826251.314194:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011387136:192.168.203.104@tcp:2:lfs.0 00000100:00100000:1.0:1787826251.315356:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011387008:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826251.317263:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011387136:192.168.203.104@tcp:2:lfs.0 00000020:00010000:3.0:1787826251.317652:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b1c68 (ffff907d884a7998/0) at cl_lock_release():245 00000020:00010000:3.0:1787826251.317728:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b1c68 (ffff907d884a7998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826251.317735:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d9862c7b0 (ffff907d884a7ab8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787826251.317749:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1dd66eb/0xe5f736940357b60b lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc0f535 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.317768:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1dd66eb/0xe5f736940357b60b lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0f535 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:3.0:1787826251.317799:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d9862c7f0 (ffff907d884a7488/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787826251.317804:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001b7e234c/0xe5f736940357b612 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc0f53c expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.317813:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001b7e234c/0xe5f736940357b612 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0f53c expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:3.0:1787826251.317822:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b1c68 (ffff907d884a7998/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826251.317825:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d9862c7b0 (ffff907d884a7998/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826251.317831:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d9862c7f0 (ffff907d884a7998/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826251.317855:0:65313:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fb:0x0] ignore/verify layout 0/0, layout version 1 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:3.0:1787826251.317873:0:65313:0:(llite_lib.c:2686:ll_setattr_raw()) COMPLETED file . :VOLATILE:0000:1B88D17B:fd=03:[0x200000404:0x1fb:0x0](ffff907d9221cb08) current size 0, valid attrs 100101f0, mode 907d, uid 0, gid 0, new size 4294967296, atime 1787826250.000000000, mtime 1787826250.000000000, ctime 1787826251.000000000, rc 0 00000080:00200008:3.0:1787826251.317965:0:65313:0:(llite_lib.c:2427:ll_setattr_raw()) START file f203.sanity-flr:[0x200000404:0x1f9:0x0](ffff907d84541988) current size 0, valid attrs 101f0, mode 907d, uid 0, gid 0, new size 0, atime 1787826250.000000000, mtime 1787826250.000000000, ctime 1787826251.000000000 00010000:00010000:3.0:1787826251.318004:0:65313:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1787826251.318032:0:65313:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f3f3 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1787826251.318076:0:65313:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 1/0,0 mode: --/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1787826251.318082:0:65313:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787826251.318085:0:65313:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b5cc lrc: 0/0,0 mode: --/PR res: [0x200000404:0x1f9:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0f4a2 expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000020:00100000:3.0:1787826251.318107:0:65313:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1787826251.318119:0:65313:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83524000 x1874670011387520/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826251.318145:0:65313:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011387520:192.168.203.104@tcp:36:lfs.0 00000100:00100000:3.0:1787826251.318263:0:65313:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d82fb9c00 going to sleep for 13 seconds 00000100:00100000:3.0:1787826251.322380:0:65313:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011387520:192.168.203.104@tcp:36:lfs.0 00000080:00200000:3.0:1787826251.322465:0:65313:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/0, layout version 1 restore needed 0 00020000:00200000:3.0:1787826251.322488:0:65313:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787826251.322509:0:65313:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:3.0:1787826251.322530:0:65313:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787826251.322536:0:65313:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d98a1c7b0, osc lock ffff907d963b94b8, flags 0x0 00010000:00010000:3.0:1787826251.322563:0:65313:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d91e6e400 type 11 mode 2 res 9663677440/801 (0 18446744073709551615) 00010000:00010000:3.0:1787826251.322673:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b619 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x321:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.322687:0:65313:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b619 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x321:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.322699:0:65313:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b619 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x321:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826251.322723:0:65313:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011387648:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826251.322814:0:65313:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d82fb9c00 going to sleep for 11 seconds 00000100:00100000:2.0:1787826251.326476:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826251.326589:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025478016 00000100:00100000:3.0:1787826251.326602:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025478016 00000100:00100000:2.0:1787826251.327052:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7121 00000100:00100000:2.0:1787826251.327065:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91148f00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:6932:x1874670025478016:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826251.327090:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b5e1 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x321:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0f4da expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1787826251.327197:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826251.327205:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91148f00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:6932:x1874670025478016:12345-192.168.203.104@tcp:104: Request processed in 143us (879us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826251.327214:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7121 00000100:00100000:2.0:1787826251.330288:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826251.330567:0:40338:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:3.0:1787826251.330591:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025478144 00000100:00100000:3.0:1787826251.330603:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025478144 00000100:00100000:3.0:1787826251.330763:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7122 00000100:00100000:3.0:1787826251.330774:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83faf850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:5934:x1874670025478144:12345-192.168.203.104@tcp:105: 00010000:00010000:3.0:1787826251.330792:0:40338:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b619 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f54a expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.330808:0:40338:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b619 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f54a expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.330818:0:40338:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b619 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f54a expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.330830:0:40338:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b619 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10080000000000 nid: local remote: 0x7918213f4fc0f54a expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.330841:0:40338:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b619 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10080000000000 nid: local remote: 0x7918213f4fc0f54a expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.330852:0:40338:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b619 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80000000000 nid: local remote: 0x7918213f4fc0f54a expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826251.330871:0:40338:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff907d86042800) 00000100:00100000:3.0:1787826251.330939:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83faf850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:5934:x1874670025478144:12345-192.168.203.104@tcp:105: Request processed in 167us (664us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826251.330946:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7122 00000100:00100000:1.0:1787826251.331345:0:65313:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011387648:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826251.331367:0:65313:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b619 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0f54a expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.331387:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b619 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc0f54a expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826251.331401:0:65313:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b619 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000020000 nid: local remote: 0x7918213f4fc0f54a expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.331477:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b619 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc0f54a expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.331491:0:65313:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b619 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc0f54a expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00010000:1.0:1787826251.336547:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b1c68 (ffff907d884a7998/0) at cl_lock_release():245 00000020:00010000:1.0:1787826251.336552:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b1c68 (ffff907d884a7998/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826251.336558:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98a1c7b0 (ffff907d884a7488/1) at cl_lock_cancel():142 00010000:00010000:1.0:1787826251.336571:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b619 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc0f54a expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.336589:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b619 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0f54a expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:1.0:1787826251.336648:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b1c68 (ffff907d884a7998/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826251.336651:0:65313:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98a1c7b0 (ffff907d884a7998/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826251.336675:0:65313:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f9:0x0] ignore/verify layout 0/0, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826251.336692:0:65313:0:(llite_lib.c:2686:ll_setattr_raw()) COMPLETED file f203.sanity-flr:[0x200000404:0x1f9:0x0](ffff907d84541988) current size 0, valid attrs 100101f0, mode 907d, uid 0, gid 0, new size 0, atime 1787826250.000000000, mtime 1787826250.000000000, ctime 1787826251.000000000, rc 0 00000080:00200080:1.0:1787826251.336722:0:65313:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1f9:0x0](ffff907d84541988) cmd=c00c66f3 arg=1dc2470 00010000:00010000:1.0:1787826251.336954:0:65313:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b5fd lrc: 2/0,0 mode: PW/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81008000000000 nid: local remote: 0x7918213f4fc0f504 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.337046:0:65313:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da563b600/0xe5f736940357b5fd lrc: 0/0,0 mode: --/PW res: [0x200000404:0x1f9:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x85809400000000 nid: local remote: 0x7918213f4fc0f504 expref: -99 pid: 65313 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826251.337071:0:65313:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d911483c0 x1874670011385344/t8589942925(8589942925) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 656/744 e 0 to 0 dl 1787826269 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826251.337083:0:65313:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826251.337089:0:65313:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a03c0 x1874670011388160/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 504/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826251.337116:0:65313:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011388160:192.168.203.104@tcp:35:lfs.0 00000100:00100000:1.0:1787826251.337340:0:65313:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d85569300 going to sleep for 11 seconds 00000100:00100000:2.0:1787826251.344232:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826251.344276:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025478272 00000100:00100000:2.0:1787826251.344287:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025478272 00000100:00100000:2.0:1787826251.344368:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7123 00000100:00100000:2.0:1787826251.344379:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d9114b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32459:x1874670025478272:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826251.345028:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357b5a9 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0f3f3 expref: -99 pid: 65309 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826251.345142:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826251.345153:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d9114b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32459:x1874670025478272:12345-192.168.203.104@tcp:104: Request processed in 774us (933us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826251.345162:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7123 00000100:00100000:2.0:1787826251.351877:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826251.352231:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025478400 00000100:00100000:3.0:1787826251.352244:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025478400 00000100:00100000:3.0:1787826251.352328:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7124 00000100:00100000:3.0:1787826251.352350:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91148780 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32459:x1874670025478400:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826251.352378:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b604 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fb:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0f52e expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826251.352496:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826251.352504:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91148780 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32459:x1874670025478400:12345-192.168.203.104@tcp:104: Request processed in 166us (641us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826251.352512:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7124 00010000:00010000:1.0:1787826251.352565:0:60835:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826251.352576:0:60835:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b604 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fb:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0f52e expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826251.352604:0:60835:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a200 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826251.352649:0:60835:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b604 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fb:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc0f52e expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826251.352659:0:60835:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b604 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fb:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc0f52e expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826251.352674:0:60835:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b604 lrc: 3/0,0 mode: CR/CR res: [0x200000404:0x1fb:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x7918213f4fc0f52e expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826251.352710:0:60835:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x1fb:0x0]: losing layout lock 00010000:00010000:1.0:1787826251.352769:0:60835:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b604 lrc: 2/0,0 mode: --/CR res: [0x200000404:0x1fb:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0f52e expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826251.352777:0:60835:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826251.352799:0:60835:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b604 lrc: 1/0,0 mode: --/CR res: [0x200000404:0x1fb:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0f52e expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826251.352808:0:60835:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b604 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1fb:0x0].0x0 bits 0x8/0x8 rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0f52e expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826251.352844:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826251.352847:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826251.352848:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826251.352858:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1787826251.352877:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011388416:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826251.355564:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011388416:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826251.361734:0:65313:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011388160:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826251.361772:0:65313:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d911483c0 x1874670011385344/t8589942925(8589942925) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 656/744 e 0 to 0 dl 1787826269 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1787826251.361821:0:65313:0:(file.c:7076:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000404:0x1f9:0x0](ffff907d84541988) 00000002:00010000:1.0:1787826251.361833:0:65313:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1f9:0x0]) in obj [0x200000404:0x1f9:0x0], intent: layout flags 0 00010000:00010000:1.0:1787826251.361884:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b620 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.361905:0:65313:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b620 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.361915:0:65313:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b620 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65313 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826251.361935:0:65313:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011388544:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826251.362133:0:65313:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d85569300 going to sleep for 13 seconds 00000100:00100000:1.0:1787826251.364847:0:65313:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011388544:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826251.364893:0:65313:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b620 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f55f expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.364911:0:65313:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b620 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f55f expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826251.364931:0:65313:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x400 disposition=0x1, status=0 req@ffff907d908a03c0 x1874670011388544/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/608 e 0 to 0 dl 1787826269 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826251.364943:0:65313:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 264 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b620 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f55f expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826251.364958:0:65313:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b620 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f55f expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.364974:0:65313:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/505 (0 0) 00000080:00200000:1.0:1787826251.364982:0:65313:0:(namei.c:952:ll_intent_lock()) intent lock 1024 on i1 [0x200000404:0x1f9:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1787826251.365004:0:65313:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1f9:0x0](ffff907d84541988) for lock 0xe5f736940357b620 00000080:00010000:1.0:1787826251.365014:0:65313:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b620 from it ffff9dd141853bc8 00010000:00010000:1.0:1787826251.365020:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b620 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f55f expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.365031:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b620 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f55f expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826251.365059:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b620 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f55f expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826251.365072:0:65313:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b620 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f55f expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:1.0:1787826251.365080:0:65313:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x1f9:0x0](00000000ce105185) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b620 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f55f expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826251.365134:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b620 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f55f expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826251.365143:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b620 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f55f expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826251.365164:0:65313:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826251.365180:0:65313:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1787826251.365230:0:65313:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1787826251.365244:0:65313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b620 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f55f expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826251.365257:0:65313:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b620 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f55f expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:1.0:1787826251.365265:0:65313:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x1f9:0x0](00000000ce105185) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b620 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f55f expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826251.365282:0:65313:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826251.365287:0:65313:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1787826251.365306:0:65313:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1787826251.365322:0:65313:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 0 blocks 0 [cma]time 1787826251 1787826250 1787826250 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b619 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x321:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0f54a expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826251.365426:0:65313:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1f9:0x0]: layout version change: 4294967294 -> 4 00010000:00010000:1.0:1787826251.365432:0:65313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b620 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f55f expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826251.365440:0:65313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a800/0xe5f736940357b620 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1f9:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f55f expref: -99 pid: 65313 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:1.0:1787826251.365486:0:65313:0:(file.c:479:ll_file_release()) START file f203.sanity-flr:[0x200000404:0x1f9:0x0](ffff907d84541988), flags 1154002 00000002:00100000:1.0:1787826251.365552:0:65313:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d908a1a40 x1874670011383680/t8589942921(8589942921) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 664/744 e 0 to 0 dl 1787826269 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826251.365565:0:65313:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826251.365572:0:65313:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a03c0 x1874670011388800/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826251.365658:0:65313:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011388800:192.168.203.104@tcp:35:lfs.0 00000100:00100000:1.0:1787826251.365772:0:65313:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d85569300 going to sleep for 11 seconds 00000100:00100000:1.0:1787826251.369315:0:65313:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011388800:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826251.369371:0:65313:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d908a1a40 x1874670011383680/t8589942921(8589942921) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 664/744 e 0 to 0 dl 1787826269 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1787826251.369467:0:65313:0:(file.c:479:ll_file_release()) START file . :VOLATILE:0000:1B88D17B:fd=03:[0x200000404:0x1fb:0x0](ffff907d9221cb08), flags 554001 00000002:00100000:1.0:1787826251.369534:0:65313:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d908a0b40 x1874670011383936/t8589942922(8589942922) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 552/608 e 0 to 0 dl 1787826269 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826251.369545:0:65313:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826251.369551:0:65313:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a03c0 x1874670011389056/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826251.369577:0:65313:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011389056:192.168.203.104@tcp:35:lfs.0 00000100:00100000:1.0:1787826251.369740:0:65313:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d85569300 going to sleep for 11 seconds 00000100:00100000:1.0:1787826251.373659:0:65313:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65313:1874670011389056:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826251.373701:0:65313:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d908a0b40 x1874670011383936/t8589942922(8589942922) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 552/608 e 0 to 0 dl 1787826269 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826251.383257:0:65314:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826251.383297:0:65314:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826251.383378:0:65314:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 65314 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.383390:0:65314:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65314 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.383407:0:65314:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65314 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826251.383440:0:65314:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65314:1874670011389184:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826251.383644:0:65314:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d85569300 going to sleep for 13 seconds 00000100:00100000:1.0:1787826251.386715:0:65314:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65314:1874670011389184:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826251.386750:0:65314:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.386765:0:65314:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.386778:0:65314:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.386790:0:65314:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.386807:0:65314:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.386815:0:65314:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.386822:0:65314:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826251.386842:0:65314:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d908a0b40 x1874670011389184/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/688 e 0 to 0 dl 1787826269 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826251.386858:0:65314:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.386874:0:65314:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1787826251.386881:0:65314:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826251.386915:0:65314:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:1.0:1787826251.386922:0:65314:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b627 00000080:00010000:1.0:1787826251.386929:0:65314:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826251.386932:0:65314:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b627 from it ffff9dd140d4bbf8 00010000:00010000:1.0:1787826251.386935:0:65314:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.386948:0:65314:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826251.386977:0:65314:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826251.387008:0:65314:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), file ffff907d91606a80, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1787826251.387016:0:65314:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=open 00000002:00010000:1.0:1787826251.387043:0:65314:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f203.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:1.0:1787826251.387102:0:65314:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b62e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65314 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.387109:0:65314:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b62e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65314 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826251.387126:0:65314:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826251.387132:0:65314:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0b40 x1874670011389440/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826251.387142:0:65314:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b62e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65314 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826251.387163:0:65314:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65314:1874670011389440:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826251.387392:0:65314:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d85569300 going to sleep for 13 seconds 00000100:00100000:1.0:1787826251.391822:0:65314:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65314:1874670011389440:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826251.391855:0:65314:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b62e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65314 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.391872:0:65314:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b62e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65314 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.391879:0:65314:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b62e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65314 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.391887:0:65314:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b62e lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65314 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.391897:0:65314:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b62e lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65314 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.391904:0:65314:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0b400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826251.391909:0:65314:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b62e lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 65314 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.391923:0:65314:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b62e lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 65314 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.391931:0:65314:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b62e lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 65314 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.391939:0:65314:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826251.391948:0:65314:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b62e lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65314 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.391955:0:65314:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b62e lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65314 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826251.391977:0:65314:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908a0b40 x1874670011389440/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/608 e 0 to 0 dl 1787826269 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826251.391994:0:65314:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826251.392180:0:65314:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826251.392216:0:65314:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826251.392234:0:65314:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826251.392249:0:65314:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826251.392257:0:65314:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826251.392264:0:65314:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826251.392268:0:65314:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b627 00000080:00010000:1.0:1787826251.392274:0:65314:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b627 from it ffff9dd140d4bbb0 00010000:00010000:1.0:1787826251.392279:0:65314:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826251.392288:0:65314:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826251.392306:0:65314:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:1.0:1787826251.392319:0:65314:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00000080:00200080:1.0:1787826251.392363:0:65314:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=c0086915 arg=1d2e3d0 00000100:00100000:1.0:1787826251.392443:0:65314:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65314:1874670011389568:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826251.392561:0:65314:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d85569300 going to sleep for 13 seconds 00000100:00100000:1.0:1787826251.396347:0:65314:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65314:1874670011389568:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826251.396729:0:65314:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00000080:00200008:2.0:1787826251.423183:0:65315:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00000080:00200000:1.0:1787826251.441581:0:65316:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826251.442338:0:65316:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826251.442358:0:65316:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826251.442374:0:65316:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1787826251.442383:0:65316:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826251.442391:0:65316:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826251.442394:0:65316:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b627 00000080:00010000:1.0:1787826251.442402:0:65316:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b627 from it ffff9dd141c47bf8 00010000:00010000:1.0:1787826251.442406:0:65316:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826251.442419:0:65316:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1787826251.442433:0:65316:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826251.444466:0:65316:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), file ffff907d90bfd500, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1787826251.444482:0:65316:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=open 00000002:00010000:1.0:1787826251.444509:0:65316:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f203.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:1.0:1787826251.444582:0:65316:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b63c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65316 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.444603:0:65316:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b63c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65316 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826251.444643:0:65316:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826251.444656:0:65316:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0b40 x1874670011390208/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826251.444667:0:65316:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b63c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65316 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826251.444691:0:65316:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65316:1874670011390208:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826251.444905:0:65316:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d85569300 going to sleep for 13 seconds 00000100:00100000:1.0:1787826251.450463:0:65316:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65316:1874670011390208:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826251.450494:0:65316:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b63c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65316 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.450508:0:65316:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b63c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65316 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.450515:0:65316:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b63c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65316 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.450527:0:65316:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b63c lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65316 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.450537:0:65316:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b63c lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65316 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.450543:0:65316:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0b400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826251.450548:0:65316:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b63c lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 65316 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.450555:0:65316:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b63c lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 65316 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.450562:0:65316:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b63c lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 65316 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.450569:0:65316:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826251.450576:0:65316:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b63c lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65316 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.450593:0:65316:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b63c lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65316 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826251.450667:0:65316:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908a0b40 x1874670011390208/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/608 e 0 to 0 dl 1787826269 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826251.450682:0:65316:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826251.450814:0:65316:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826251.450843:0:65316:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826251.450856:0:65316:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826251.450868:0:65316:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1787826251.450875:0:65316:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826251.450880:0:65316:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826251.450884:0:65316:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b627 00000080:00010000:1.0:1787826251.450889:0:65316:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b627 from it ffff9dd141c47bb0 00010000:00010000:1.0:1787826251.450892:0:65316:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826251.450900:0:65316:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1787826251.450914:0:65316:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:1.0:1787826251.450924:0:65316:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00000080:00200080:1.0:1787826251.450957:0:65316:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=c0086915 arg=1bff3d0 00000100:00100000:1.0:1787826251.451012:0:65316:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65316:1874670011390336:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826251.451093:0:65316:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d85569300 going to sleep for 13 seconds 00000100:00100000:1.0:1787826251.457283:0:65316:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65316:1874670011390336:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826251.458058:0:65316:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00000100:00100000:1.0:1787826251.503037:0:65317:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65317:1874670011390592:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826251.503070:0:65317:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b643 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65317 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.503089:0:65317:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b643 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65317 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.503097:0:65317:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b643 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65317 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.503105:0:65317:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b643 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65317 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.503114:0:65317:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b643 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65317 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.503121:0:65317:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a200 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826251.503126:0:65317:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b643 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 65317 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.503134:0:65317:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b643 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 65317 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.503142:0:65317:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b643 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 65317 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.503150:0:65317:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826251.503159:0:65317:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b643 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65317 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826251.503166:0:65317:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b643 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65317 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826251.503188:0:65317:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908461c0 x1874670011390592/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2232/608 e 0 to 0 dl 1787826269 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826251.503213:0:65317:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826251.503384:0:65317:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826251.503409:0:65317:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826251.503429:0:65317:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826251.503442:0:65317:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1787826251.503449:0:65317:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826251.503457:0:65317:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826251.503461:0:65317:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b627 00000080:00010000:1.0:1787826251.503468:0:65317:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b627 from it ffff9dd141c4fbb0 00010000:00010000:1.0:1787826251.503471:0:65317:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826251.503480:0:65317:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1787826251.503495:0:65317:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:1.0:1787826251.503508:0:65317:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00000080:00200080:1.0:1787826251.503551:0:65317:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=c0086915 arg=11603d0 00000100:00100000:1.0:1787826251.503667:0:65317:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65317:1874670011390720:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826251.503798:0:65317:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d85569300 going to sleep for 13 seconds 00000100:00100000:3.0:1787826251.509190:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1787826251.509224:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011390848:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826251.514506:0:65317:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65317:1874670011390720:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826251.519087:0:65317:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 304000 00000100:00100000:2.0:1787826251.520742:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011390848:192.168.203.104@tcp:400:kworker.0 00010000:00010000:1.0:1787826251.636905:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1787826253.560200:0:61332:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f7:0x0] ignore/verify layout 1/0, layout version 3 restore needed 0 00020000:00200000:2.0:1787826253.560215:0:61332:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1787826253.560224:0:61332:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00000008:00010000:2.0:1787826253.560249:0:61332:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:2.0:1787826253.560267:0:61332:0:(osc_cache.c:1174:osc_extent_make_ready()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826253.560538:0:61332:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 1052671) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00100000:2.0:1787826253.561084:0:61332:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d91148f00 - object 0x280000400:772 offset 1048576<>2097152 00000100:00100000:2.0:1787826253.561193:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011390976:192.168.203.104@tcp:4:dd.0 00000100:00100000:2.0:1787826253.561295:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1787826253.561302:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011391104:192.168.203.104@tcp:16:kworker.0 00000100:00100000:2.0:1787826253.578232:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011391104:192.168.203.104@tcp:16:kworker.0 00000080:00200000:3.0:1787826253.578338:0:61332:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f7:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787826253.578358:0:61332:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fb:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1787826253.578369:0:61332:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787826253.578389:0:61332:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fb:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787826253.578405:0:61332:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1787826253.578411:0:61332:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1f9:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:3.0:1787826253.578414:0:61332:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:3.0:1787826253.578417:0:61332:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787826253.578423:0:61332:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:3.0:1787826253.613342:0:2350:0:(osc_cache.c:908:osc_extent_finish()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:3.0:1787826253.613996:0:2350:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 000000007bf5a8e9 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000100:00100000:3.0:1787826253.614034:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011390976:192.168.203.104@tcp:4:dd.0 00010000:00010000:2.0:1787826253.686974:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826254.711054:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826255.741773:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826256.628871:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011391232:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826256.628968:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83526580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011391488:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826256.629138:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1787826256.629150:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835243c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011391360:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826256.629181:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011391616:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826256.634828:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835243c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011391360:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826256.634849:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011391616:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826256.634861:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83526580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011391488:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826256.641826:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011391232:192.168.203.104@tcp:400:kworker.0 00010000:00010000:1.0:1787826256.759643:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1787826258.332136:0:65873:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-flr test 204a: FLR write/stale/resync tests with self-extending mirror ========================================================== 06:24:16 (1787826256) 00000080:00200000:3.0:1787826258.637639:0:65892:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826258.637665:0:65892:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826258.637682:0:65892:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1787826258.637699:0:65892:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:3.0:1787826258.637712:0:65892:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826258.637720:0:65892:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826258.637724:0:65892:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b627 00000080:00010000:3.0:1787826258.637732:0:65892:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b627 from it ffff9dd14186fbe0 00010000:00010000:3.0:1787826258.637737:0:65892:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1787826258.637749:0:65892:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:3.0:1787826258.637768:0:65892:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826258.637784:0:65892:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d204a.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), flags=1536 00000080:00200000:3.0:1787826258.637788:0:65892:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826258.637794:0:65892:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826258.637800:0:65892:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:3.0:1787826258.637809:0:65892:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:3.0:1787826258.637816:0:65892:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826258.637820:0:65892:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826258.637822:0:65892:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b627 00000080:00010000:3.0:1787826258.637826:0:65892:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b627 from it ffff9dd14186fce0 00010000:00010000:3.0:1787826258.637829:0:65892:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:3.0:1787826258.637843:0:65892:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:3.0:1787826258.637852:0:65892:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 3 00000080:00200000:3.0:1787826258.637857:0:65892:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826258.637863:0:65892:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826258.637869:0:65892:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:3.0:1787826258.637877:0:65892:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:3.0:1787826258.637884:0:65892:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826258.637888:0:65892:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826258.637890:0:65892:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b627 00000080:00010000:3.0:1787826258.637894:0:65892:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b627 from it ffff9dd14186fdc8 00010000:00010000:3.0:1787826258.637897:0:65892:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:3.0:1787826258.637905:0:65892:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:3.0:1787826258.637913:0:65892:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 3 00000080:00200000:3.0:1787826258.637919:0:65892:0:(namei.c:2239:do_mkdir()) VFS Op:name=d204a.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148) 00010000:00010000:3.0:1787826258.637955:0:65892:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:3.0:1787826258.638021:0:65892:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:3.0:1787826258.638029:0:65892:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787826258.638033:0:65892:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b627 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0f574 expref: -99 pid: 65314 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000020:00100000:3.0:1787826258.638061:0:65892:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1787826258.638075:0:65892:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d835243c0 x1874670011391872/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 568/2152 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826258.638105:0:65892:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835243c0 pname:cluuid:pid:xid:nid:opc:job mkdir:6f891bcb-15cd-41e0-96e2-db843619c1be:65892:1874670011391872:192.168.203.104@tcp:36:mkdir.0 00000100:00100000:3.0:1787826258.638224:0:65892:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d960579c0 going to sleep for 13 seconds 00000100:00100000:2.0:1787826258.651691:0:65892:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589942932 gen 1 00000100:00100000:2.0:1787826258.651727:0:65892:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff907d835243c0 x1874670011391872/t8589942933(8589942933) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 568/448 e 0 to 0 dl 1787826276 ref 3 fl Rpc:RQU/204/0 rc 0/0 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787826258.651741:0:65892:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835243c0 pname:cluuid:pid:xid:nid:opc:job mkdir:6f891bcb-15cd-41e0-96e2-db843619c1be:65892:1874670011391872:192.168.203.104@tcp:36:mkdir.0 00000080:00200000:2.0:1787826258.651779:0:65892:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8) 00000080:00200000:2.0:1787826258.651785:0:65892:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x1fc:0x0], updating i_size 4096 00000080:00200000:2.0:1787826258.689047:0:65895:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826258.689074:0:65895:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826258.689143:0:65895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 65895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826258.689154:0:65895:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826258.689164:0:65895:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65895 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826258.689191:0:65895:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65895:1874670011392000:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826258.689298:0:65895:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6540 going to sleep for 13 seconds 00000100:00100000:3.0:1787826258.695635:0:65895:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65895:1874670011392000:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826258.695664:0:65895:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826258.695677:0:65895:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826258.695787:0:65895:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826258.695800:0:65895:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826258.695814:0:65895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826258.695831:0:65895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826258.695839:0:65895:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826258.695851:0:65895:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d9114a1c0 x1874670011392000/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/688 e 0 to 0 dl 1787826276 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1787826258.695866:0:65895:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826258.695879:0:65895:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1787826258.695884:0:65895:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1787826258.696007:0:65895:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:2.0:1787826258.696015:0:65895:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:2.0:1787826258.696022:0:65895:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826258.696025:0:65895:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141fa3b58 00010000:00010000:2.0:1787826258.696028:0:65895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826258.696038:0:65895:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826258.696062:0:65895:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826258.696080:0:65895:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d204a.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), flags=29 00000080:00200000:2.0:1787826258.696084:0:65895:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d204a.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=0 00000002:00010000:2.0:1787826258.696101:0:65895:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: d204a.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00010000:00010000:2.0:1787826258.696140:0:65895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62ea800/0xe5f736940357b651 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826258.696147:0:65895:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62ea800/0xe5f736940357b651 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826258.696161:0:65895:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62ea800/0xe5f736940357b651 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65895 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826258.696180:0:65895:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65895:1874670011392128:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826258.696259:0:65895:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f66c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826258.752963:0:65895:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88867d80 going to sleep for 13 seconds 00000100:00100000:1.0:1787826258.760758:0:65895:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65895:1874670011392896:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826258.760786:0:65895:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b674 lrc: 4/0,1 mode: --/CW res: [0x200000404:0x1fc:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f6bd expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.760809:0:65895:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b674 lrc: 4/0,1 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f6bd expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.760818:0:65895:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b674 lrc: 4/0,1 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f6bd expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.760831:0:65895:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b674 lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f6bd expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.760842:0:65895:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b674 lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f6bd expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.760856:0:65895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b674 lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f6bd expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.760864:0:65895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b674 lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f6bd expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.760872:0:65895:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b674 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f6bd expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826258.760889:0:65895:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x8000037, status=0 req@ffff907d908443c0 x1874670011392896/t8589942934(8589942934) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826276 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826258.760902:0:65895:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908443c0 x1874670011392896/t8589942934(8589942934) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826276 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826258.760914:0:65895:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b674 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f6bd expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.760925:0:65895:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/509 (0 0) 00000080:00200000:1.0:1787826258.760932:0:65895:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x200000404:0x1fc:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1787826258.760944:0:65895:0:(namei.c:1628:ll_create_it()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), intent=open|creat 00000080:00200000:1.0:1787826258.760974:0:65895:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148) 00000080:00200000:1.0:1787826258.760981:0:65895:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x1fd:0x0], updating i_size 0 00000080:00200000:1.0:1787826258.761041:0:65895:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1fd:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:1.0:1787826258.761046:0:65895:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) 00000080:00010000:1.0:1787826258.761049:0:65895:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) for lock 0xe5f736940357b674 00000080:00010000:1.0:1787826258.761061:0:65895:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) 00000080:00200008:1.0:1787826258.761075:0:65895:0:(file.c:1090:ll_file_open()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0](ffff907d92219148), flags 120501 00000002:00100000:1.0:1787826258.761095:0:65895:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908443c0 x1874670011392896/t8589942934(8589942934) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826276 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1787826258.761126:0:65895:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b674 from it ffff907d832413c0 00010000:00010000:1.0:1787826258.761131:0:65895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b674 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f6bd expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.761139:0:65895:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b674 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f6bd expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826258.761160:0:65895:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148), inode mode 81a4 mask 40 00000080:00200000:1.0:1787826258.761213:0:65895:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148), inode mode 81a4 mask 2 00000080:00200000:1.0:1787826258.761223:0:65895:0:(xattr.c:414:ll_xattr_set()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148), xattr lov 00000002:00010000:1.0:1787826258.761239:0:65895:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1fd:0x0]) in obj [0x200000404:0x1fc:0x0], intent: open flags 040000002 00010000:00010000:1.0:1787826258.761298:0:65895:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b67b lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.761306:0:65895:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b67b lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65895 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826258.761316:0:65895:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826258.761323:0:65895:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a2940 x1874670011393152/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826258.761332:0:65895:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b67b lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65895 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826258.761352:0:65895:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65895:1874670011393152:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826258.763389:0:65895:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb600 going to sleep for 13 seconds 00000100:00100000:3.0:1787826258.777049:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826258.777212:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025480192 00000100:00100000:2.0:1787826258.777235:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025480192 00000100:00100000:2.0:1787826258.777292:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7125 00000100:00100000:2.0:1787826258.777301:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83527c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:35508:x1874670025480192:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826258.777318:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b674 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0f6bd expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826258.777395:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826258.777402:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83527c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:35508:x1874670025480192:12345-192.168.203.104@tcp:104: Request processed in 103us (363us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826258.777409:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7125 00010000:00010000:1.0:1787826258.777492:0:60835:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826258.777500:0:60835:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b674 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0f6bd expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826258.777511:0:60835:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a200 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826258.777517:0:60835:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b674 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc0f6bd expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826258.777526:0:60835:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b674 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc0f6bd expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826258.777535:0:60835:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b674 lrc: 3/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x7918213f4fc0f6bd expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826258.777579:0:60835:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x1fd:0x0]: losing layout lock 00010000:00010000:1.0:1787826258.777926:0:60835:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b674 lrc: 2/0,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0f6bd expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826258.777935:0:60835:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826258.778085:0:60835:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b674 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0f6bd expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826258.778094:0:60835:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b674 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0f6bd expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826258.778114:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826258.778116:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826258.778118:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826258.786266:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011393280:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826258.788003:0:65895:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65895:1874670011393152:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826258.788025:0:65895:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b67b lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.788038:0:65895:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b67b lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.788046:0:65895:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b67b lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.788053:0:65895:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b67b lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.788062:0:65895:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b67b lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.788068:0:65895:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0b800 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826258.788072:0:65895:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b67b lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 65895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.788078:0:65895:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b67b lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 65895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.788085:0:65895:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b67b lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 65895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.788092:0:65895:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826258.788099:0:65895:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b67b lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65895 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.788105:0:65895:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b67b lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65895 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826258.788125:0:65895:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x800002b, status=0 req@ffff907d908a2940 x1874670011393152/t8589942935(8589942935) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1160 e 0 to 0 dl 1787826276 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826258.788139:0:65895:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a2940 x1874670011393152/t8589942935(8589942935) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1160 e 0 to 0 dl 1787826276 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826258.788156:0:65895:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1fc:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826258.788204:0:65895:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1787826258.788213:0:65895:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826258.788254:0:65895:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x1fd:0x0], updating i_size 0 00000002:00100000:1.0:1787826258.788269:0:65895:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a2940 x1874670011393152/t8589942935(8589942935) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826276 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826258.788303:0:65895:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d908a2940 x1874670011393152/t8589942935(8589942935) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826276 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826258.788313:0:65895:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826258.788318:0:65895:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a2d00 x1874670011393536/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826258.788339:0:65895:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65895:1874670011393536:192.168.203.104@tcp:35:lfs.0 00000100:00100000:1.0:1787826258.789330:0:65895:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb600 going to sleep for 11 seconds 00000100:00100000:1.0:1787826258.792104:0:65895:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65895:1874670011393536:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826258.792138:0:65895:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d908a2940 x1874670011393152/t8589942935(8589942935) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826276 ref 2 fl Complete:RQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1787826258.792203:0:65895:0:(file.c:479:ll_file_release()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0](ffff907d92219148), flags 100001 00000002:00100000:1.0:1787826258.792252:0:65895:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d908443c0 x1874670011392896/t8589942934(8589942934) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826276 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826258.792261:0:65895:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826258.792266:0:65895:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a1680 x1874670011393792/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826258.792286:0:65895:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65895:1874670011393792:192.168.203.104@tcp:35:lfs.0 00000100:00100000:1.0:1787826258.792365:0:65895:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb600 going to sleep for 11 seconds 00000100:00100000:1.0:1787826258.794344:0:65895:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65895:1874670011393792:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826258.794373:0:65895:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d908443c0 x1874670011392896/t8589942934(8589942934) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826276 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826258.800131:0:65896:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826258.800159:0:65896:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826258.800178:0:65896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826258.800193:0:65896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1787826258.800201:0:65896:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826258.800208:0:65896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826258.800211:0:65896:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:1.0:1787826258.800219:0:65896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141883bf8 00010000:00010000:1.0:1787826258.800222:0:65896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826258.800229:0:65896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1787826258.800243:0:65896:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826258.800250:0:65896:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=25 00000080:00200000:1.0:1787826258.800254:0:65896:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 1 00000080:00200000:1.0:1787826258.800265:0:65896:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 3 00000080:00200000:1.0:1787826258.800270:0:65896:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), file ffff907d82cbf980, open_flags 8041, mode 81b6 opened 0 00000080:00200000:1.0:1787826258.800277:0:65896:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), intent=open|creat 00000002:00010000:1.0:1787826258.800295:0:65896:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204a.sanity-flr,[0x200000404:0x1fe:0x0]) in obj [0x200000404:0x1fc:0x0], intent: open|creat flags 0100102 00010000:00010000:1.0:1787826258.800346:0:65896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b682 lrc: 3/0,1 mode: --/CW res: [0x200000404:0x1fc:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.800354:0:65896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b682 lrc: 3/0,1 mode: --/CW res: [0x200000404:0x1fc:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65896 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826258.800365:0:65896:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826258.800376:0:65896:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a2580 x1874670011394048/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/67896 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826258.800392:0:65896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b682 lrc: 3/0,1 mode: --/CW res: [0x200000404:0x1fc:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826258.800411:0:65896:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:65896:1874670011394048:192.168.203.104@tcp:101:dd.0 00000100:00100000:1.0:1787826258.800546:0:65896:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb600 going to sleep for 13 seconds 00000100:00100000:1.0:1787826258.803530:0:65896:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:65896:1874670011394048:192.168.203.104@tcp:101:dd.0 00010000:00010000:1.0:1787826258.803562:0:65896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b682 lrc: 4/0,1 mode: --/CW res: [0x200000404:0x1fc:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f6f5 expref: -99 pid: 65896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.803599:0:65896:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b682 lrc: 4/0,1 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f6f5 expref: -99 pid: 65896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.803638:0:65896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b682 lrc: 4/0,1 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f6f5 expref: -99 pid: 65896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.803649:0:65896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b682 lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f6f5 expref: -99 pid: 65896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.803658:0:65896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b682 lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f6f5 expref: -99 pid: 65896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.803671:0:65896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b682 lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f6f5 expref: -99 pid: 65896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.803680:0:65896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b682 lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f6f5 expref: -99 pid: 65896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.803687:0:65896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b682 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f6f5 expref: -99 pid: 65896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826258.803698:0:65896:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x2b, status=0 req@ffff907d908a2580 x1874670011394048/t8589942938(8589942938) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/1160 e 0 to 0 dl 1787826276 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826258.803711:0:65896:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a2580 x1874670011394048/t8589942938(8589942938) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/1160 e 0 to 0 dl 1787826276 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826258.803723:0:65896:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b682 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f6f5 expref: -99 pid: 65896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826258.803734:0:65896:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b682 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f6f5 expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.803744:0:65896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/509 (0 0) 00000080:00200000:1.0:1787826258.803749:0:65896:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x200000404:0x1fc:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1787826258.803797:0:65896:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x1fd:0x0](ffff907d92219148): rc = 0 00000080:00200000:1.0:1787826258.803812:0:65896:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1fd:0x0]: layout version change: 4294967294 -> 5 00000080:00010000:1.0:1787826258.803818:0:65896:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) for lock 0xe5f736940357b682 00000080:00010000:1.0:1787826258.803828:0:65896:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) 00000080:00200008:1.0:1787826258.803907:0:65896:0:(file.c:1090:ll_file_open()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0](ffff907d92219148), flags 100101 00000002:00100000:1.0:1787826258.803929:0:65896:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a2580 x1874670011394048/t8589942938(8589942938) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1088/1160 e 0 to 0 dl 1787826276 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1787826258.803959:0:65896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b682 from it ffff907da56c23c0 00010000:00010000:1.0:1787826258.803963:0:65896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b682 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f6f5 expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.803972:0:65896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b682 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f6f5 expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826258.803992:0:65896:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148), inode mode 81a4 mask 42 00010000:00010000:2.0:1787826258.805076:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1787826258.808415:0:65896:0:(file.c:2652:do_file_write_iter()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0], ppos: 0, count: 2097152 00000080:00200000:1.0:1787826258.808432:0:65896:0:(file.c:2679:do_file_write_iter()) switching to DIO 00000080:00200000:1.0:1787826258.808438:0:65896:0:(file.c:2090:ll_file_io_generic()) f204a.sanity-flr: write ppos: 0, bytes: 2097152 00000080:00200000:1.0:1787826258.808452:0:65896:0:(file.c:1848:ll_io_set_mirror()) f204a.sanity-flr: desiginated mirror: 0 00000020:00200000:1.0:1787826258.808460:0:65896:0:(cl_io.c:218:cl_io_rw_init()) header@000000007d7e8202[0x110, 2, [0x200000404:0x1fd:0x0] hash] 00000020:00200000:1.0:1787826258.808463:0:65896:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [0, 2097152) 0 0 00000080:00200000:1.0:1787826258.808469:0:65896:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 3 restore needed 0 00020000:00200000:1.0:1787826258.808484:0:65896:0:(lov_io.c:571:lov_io_mirror_init()) [0x200000404:0x1fd:0x0] need write intent for [0, 2097152) 00000020:00200000:1.0:1787826258.808488:0:65896:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826258.808494:0:65896:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 5 need write layout 1, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826258.808497:0:65896:0:(vvp_io.c:371:vvp_io_fini()) [0x200000404:0x1fd:0x0] write layout, type 2 [0x0, 0x200000) 00000080:00010000:1.0:1787826258.808508:0:65896:0:(file.c:7076:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000404:0x1fd:0x0](ffff907d92219148) 00000002:00010000:1.0:1787826258.808521:0:65896:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1fd:0x0]) in obj [0x200000404:0x1fd:0x0], intent: layout flags 02 00010000:00010000:1.0:1787826258.808545:0:65896:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b682 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0f6f5 expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826258.808579:0:65896:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x1fd:0x0]: losing layout lock 00010000:00010000:1.0:1787826258.808667:0:65896:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b682 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0f6f5 expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826258.808675:0:65896:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826258.808678:0:65896:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b682 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0f6f5 expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826258.808709:0:65896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 3/0,1 mode: --/EX res: [0x200000404:0x1fd:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 65896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.808715:0:65896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 3/0,1 mode: --/EX res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65896 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826258.808727:0:65896:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826258.808740:0:65896:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a1e00 x1874670011394304/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/2048 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826258.808748:0:65896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 3/0,1 mode: --/EX res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826258.808769:0:65896:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:65896:1874670011394304:192.168.203.104@tcp:101:dd.0 00000100:00100000:1.0:1787826258.808948:0:65896:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb600 going to sleep for 13 seconds 00000100:00100000:1.0:1787826258.823512:0:65896:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589942939 gen 1 00000100:00100000:1.0:1787826258.823715:0:65896:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d908a2580 x1874670011394048/t8589942938(8589942938) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1088/1160 e 0 to 0 dl 1787826276 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1787826258.823735:0:65896:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:65896:1874670011394304:192.168.203.104@tcp:101:dd.0 00010000:00010000:1.0:1787826258.823763:0:65896:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 4/0,1 mode: --/EX res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.823786:0:65896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 4/0,1 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.823803:0:65896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 4/0,1 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.823815:0:65896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 4/0,1 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.823832:0:65896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 5/1,1 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.823841:0:65896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 5/1,1 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.823848:0:65896:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826258.823865:0:65896:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x400 disposition=0x1, status=0 req@ffff907d908a1e00 x1874670011394304/t8589942939(8589942939) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/896 e 0 to 0 dl 1787826276 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826258.823882:0:65896:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826258.823899:0:65896:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.825414:0:65896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/509 (0 0) 00000080:00200000:1.0:1787826258.825429:0:65896:0:(namei.c:952:ll_intent_lock()) intent lock 1024 on i1 [0x200000404:0x1fd:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1787826258.825449:0:65896:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) for lock 0xe5f736940357b689 00000080:00010000:1.0:1787826258.825460:0:65896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b689 from it ffff9dd141883bb8 00010000:00010000:1.0:1787826258.825464:0:65896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.825475:0:65896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826258.825493:0:65896:0:(file.c:2240:ll_file_io_generic()) f204a.sanity-flr: 2 io complete with rc: 0, result: 0, restart: 1 00000080:00200000:1.0:1787826258.825496:0:65896:0:(file.c:2247:ll_file_io_generic()) f204a.sanity-flr: restart write from ppos=0 bytes=2097152 retries=999 ret=0: rc = 0 00000080:00200000:1.0:1787826258.825503:0:65896:0:(file.c:1848:ll_io_set_mirror()) f204a.sanity-flr: desiginated mirror: 0 00000020:00200000:1.0:1787826258.825511:0:65896:0:(cl_io.c:218:cl_io_rw_init()) header@000000007d7e8202[0x110, 2, [0x200000404:0x1fd:0x0] hash] 00000020:00200000:1.0:1787826258.825514:0:65896:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [0, 2097152) 0 0 00000080:00200000:1.0:1787826258.825519:0:65896:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 5 restore needed 0 00010000:00010000:1.0:1787826258.825536:0:65896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826258.825547:0:65896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:1.0:1787826258.825556:0:65896:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x1fd:0x0](00000000a294d51b) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826258.825980:0:65896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826258.825995:0:65896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826258.826009:0:65896:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826258.826020:0:65896:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 1, move mirror from 1 to 1, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826258.826023:0:65896:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826258.826027:0:65896:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x10 00020000:00200000:1.0:1787826258.826035:0:65896:0:(lov_io.c:1253:lov_io_iter_init()) component[4] flags 0x40 00000080:00200000:1.0:1787826258.826052:0:65896:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1787826258.826064:0:65896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826258.826074:0:65896:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:1.0:1787826258.826081:0:65896:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x1fd:0x0](00000000a294d51b) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826258.826108:0:65896:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826258.826112:0:65896:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 1, move mirror from 1 to 1, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826258.826115:0:65896:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826258.826117:0:65896:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x10 00020000:00200000:1.0:1787826258.826172:0:65896:0:(lov_io.c:1253:lov_io_iter_init()) component[4] flags 0x40 00000080:00200000:1.0:1787826258.826332:0:65896:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826258.826397:0:65896:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1fd:0x0]: layout version change: 4294967294 -> 6 00010000:00010000:1.0:1787826258.826464:0:65896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826258.826479:0:65896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00020000:00200000:1.0:1787826258.826507:0:65896:0:(lov_io.c:311:lov_io_mirror_write_intent()) [0x200000404:0x1fd:0x0]there are 0 components to be staled to modify file extent [0x0, 0xa00000), iot: 2 00020000:00200000:1.0:1787826258.826511:0:65896:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826258.826514:0:65896:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00000080:00200000:1.0:1787826258.826518:0:65896:0:(file.c:2170:ll_file_io_generic()) Range lock [0, 511] 00020000:00200000:1.0:1787826258.826525:0:65896:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 2097152) 2097152, 2097152 00020000:00200000:1.0:1787826258.826527:0:65896:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1787826258.826539:0:65896:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 511] 00000008:00010000:1.0:1787826258.826549:0:65896:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1787826258.826553:0:65896:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c32ab0, osc lock ffff907d962f5248, flags 0x0 00000080:00200000:1.0:1787826258.826562:0:65896:0:(vvp_io.c:1339:vvp_io_write_start()) f204a.sanity-flr: write [0, 2097152) append 0 append lockpos 0 00000002:00010000:1.0:1787826258.826599:0:65896:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x0:0x0:0x0]) in obj [0x200000404:0x1fd:0x0], intent: getxattr flags 0 00010000:00010000:1.0:1787826258.826682:0:65896:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b690 lrc: 3/1,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.826691:0:65896:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b690 lrc: 3/1,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65896 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.826701:0:65896:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b690 lrc: 3/1,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65896 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826258.826724:0:65896:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:65896:1874670011394432:192.168.203.104@tcp:101:dd.0 00000100:00100000:1.0:1787826258.826937:0:65896:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb600 going to sleep for 13 seconds 00000100:00100000:1.0:1787826258.832507:0:65896:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:65896:1874670011394432:192.168.203.104@tcp:101:dd.0 00010000:00010000:1.0:1787826258.832535:0:65896:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b690 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f718 expref: -99 pid: 65896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.832556:0:65896:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b690 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f718 expref: -99 pid: 65896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.832931:0:65896:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b690 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f718 expref: -99 pid: 65896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826258.832948:0:65896:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x80 disposition=0x0, status=0 req@ffff907d908a1e00 x1874670011394432/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 568/1648 e 0 to 0 dl 1787826276 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826258.832959:0:65896:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b690 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f718 expref: -99 pid: 65896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.832982:0:65896:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/509 (0 0) 00000080:00200000:1.0:1787826258.832988:0:65896:0:(namei.c:952:ll_intent_lock()) intent lock 128 on i1 [0x200000404:0x1fd:0x0] suppgids 0 -1: rc 0 00000080:00010000:1.0:1787826258.833018:0:65896:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) for lock 0xe5f736940357b690 00000080:00010000:1.0:1787826258.833025:0:65896:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b690 from it ffff9dd1418838e8 00010000:00010000:1.0:1787826258.833029:0:65896:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b690 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f718 expref: -99 pid: 65896 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.833039:0:65896:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b690 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f718 expref: -99 pid: 65896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826258.833075:0:65896:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148), size=2097152 (max 335544320), offset=0=0x0, pages 512 (max 81920), lockless, parallel, hybrid 00000020:00200000:1.0:1787826258.836529:0:65896:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 512 items remaining. 00000080:00200000:1.0:1787826258.836538:0:65896:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 2097152 count 2097152 file_offset 2097152 00000020:00010000:1.0:1787826258.836551:0:65896:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b1c68 (ffff907d884a7998/0) at cl_lock_release():245 00000020:00010000:1.0:1787826258.836555:0:65896:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b1c68 (ffff907d884a7998/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826258.836559:0:65896:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c32ab0 (ffff907d884a7488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826258.836563:0:65896:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b1c68 (ffff907d884a7998/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826258.836565:0:65896:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c32ab0 (ffff907d884a7998/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826258.836586:0:65896:0:(vvp_io.c:533:vvp_io_advance()) advancing 2097152 bytes 00000020:00200000:1.0:1787826258.836594:0:65896:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1787826258.836831:0:49858:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907d908aa0c0 00000008:00100000:1.0:1787826258.838375:0:49858:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d908a1e00 - object 0x240000400:803 offset 0<>2097152 00000100:00100000:1.0:1787826258.838445:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011394560:192.168.203.104@tcp:4:dd.0 00000100:00100000:1.0:1787826258.886140:0:2348:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff907d986d0000: committing for last_committed 21474840484 gen 3 00000008:00010000:1.0:1787826258.886244:0:2348:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907d908aa0c0 00000020:00200000:1.0:1787826258.886251:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 511 items remaining. 00000020:00200000:1.0:1787826258.886254:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 510 items remaining. 00000020:00200000:1.0:1787826258.886256:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 509 items remaining. 00000020:00200000:1.0:1787826258.886257:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 508 items remaining. 00000020:00200000:1.0:1787826258.886259:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 507 items remaining. 00000020:00200000:1.0:1787826258.886260:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 506 items remaining. 00000020:00200000:1.0:1787826258.886262:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 505 items remaining. 00000020:00200000:1.0:1787826258.886263:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 504 items remaining. 00000020:00200000:1.0:1787826258.886264:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 503 items remaining. 00000020:00200000:1.0:1787826258.886266:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 502 items remaining. 00000020:00200000:1.0:1787826258.886267:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 501 items remaining. 00000020:00200000:1.0:1787826258.886269:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 500 items remaining. 00000020:00200000:1.0:1787826258.886270:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 499 items remaining. 00000020:00200000:1.0:1787826258.886272:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 498 items remaining. 00000020:00200000:1.0:1787826258.886273:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 497 items remaining. 00000020:00200000:1.0:1787826258.886275:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 496 items remaining. 00000020:00200000:1.0:1787826258.886276:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 495 items remaining. 00000020:00200000:1.0:1787826258.886278:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 494 items remaining. 00000020:00200000:1.0:1787826258.886279:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 493 items remaining. 00000020:00200000:1.0:1787826258.886281:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 492 items remaining. 00000020:00200000:1.0:1787826258.886283:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 491 items remaining. 00000020:00200000:1.0:1787826258.886285:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 490 items remaining. 00000020:00200000:1.0:1787826258.886287:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 489 items remaining. 00000020:00200000:1.0:1787826258.886288:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 488 items remaining. 00000020:00200000:1.0:1787826258.886290:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 487 items remaining. 00000020:00200000:1.0:1787826258.886291:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 486 items remaining. 00000020:00200000:1.0:1787826258.886293:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 485 items remaining. 00000020:00200000:1.0:1787826258.886295:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 484 items remaining. 00000020:00200000:1.0:1787826258.886296:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 483 items remaining. 00000020:00200000:1.0:1787826258.886297:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 482 items remaining. 00000020:00200000:1.0:1787826258.886299:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 481 items remaining. 00000020:00200000:1.0:1787826258.886300:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 480 items remaining. 00000020:00200000:1.0:1787826258.886302:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 479 items remaining. 00000020:00200000:1.0:1787826258.886304:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 478 items remaining. 00000020:00200000:1.0:1787826258.886305:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 477 items remaining. 00000020:00200000:1.0:1787826258.886306:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 476 items remaining. 00000020:00200000:1.0:1787826258.886308:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 475 items remaining. 00000020:00200000:1.0:1787826258.886310:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 474 items remaining. 00000020:00200000:1.0:1787826258.886311:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 473 items remaining. 00000020:00200000:1.0:1787826258.886313:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 472 items remaining. 00000020:00200000:1.0:1787826258.886314:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 471 items remaining. 00000020:00200000:1.0:1787826258.886316:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 470 items remaining. 00000020:00200000:1.0:1787826258.886318:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 469 items remaining. 00000020:00200000:1.0:1787826258.886319:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 468 items remaining. 00000020:00200000:1.0:1787826258.886321:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 467 items remaining. 00000020:00200000:1.0:1787826258.886322:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 466 items remaining. 00000020:00200000:1.0:1787826258.886324:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 465 items remaining. 00000020:00200000:1.0:1787826258.886326:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 464 items remaining. 00000020:00200000:1.0:1787826258.886327:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 463 items remaining. 00000020:00200000:1.0:1787826258.886329:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 462 items remaining. 00000020:00200000:1.0:1787826258.886331:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 461 items remaining. 00000020:00200000:1.0:1787826258.886332:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 460 items remaining. 00000020:00200000:1.0:1787826258.886334:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 459 items remaining. 00000020:00200000:1.0:1787826258.886336:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 458 items remaining. 00000020:00200000:1.0:1787826258.886338:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 457 items remaining. 00000020:00200000:1.0:1787826258.886340:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 456 items remaining. 00000020:00200000:1.0:1787826258.886342:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 455 items remaining. 00000020:00200000:1.0:1787826258.886344:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 454 items remaining. 00000020:00200000:1.0:1787826258.886345:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 453 items remaining. 00000020:00200000:1.0:1787826258.886347:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 452 items remaining. 00000020:00200000:1.0:1787826258.886349:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 451 items remaining. 00000020:00200000:1.0:1787826258.886350:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 450 items remaining. 00000020:00200000:1.0:1787826258.886352:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 449 items remaining. 00000020:00200000:1.0:1787826258.886353:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 448 items remaining. 00000020:00200000:1.0:1787826258.886355:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 447 items remaining. 00000020:00200000:1.0:1787826258.886357:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 446 items remaining. 00000020:00200000:1.0:1787826258.886359:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 445 items remaining. 00000020:00200000:1.0:1787826258.886361:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 444 items remaining. 00000020:00200000:1.0:1787826258.886363:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 443 items remaining. 00000020:00200000:1.0:1787826258.886364:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 442 items remaining. 00000020:00200000:1.0:1787826258.886366:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 441 items remaining. 00000020:00200000:1.0:1787826258.886368:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 440 items remaining. 00000020:00200000:1.0:1787826258.886370:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 439 items remaining. 00000020:00200000:1.0:1787826258.886372:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 438 items remaining. 00000020:00200000:1.0:1787826258.886373:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 437 items remaining. 00000020:00200000:1.0:1787826258.886375:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 436 items remaining. 00000020:00200000:1.0:1787826258.886376:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 435 items remaining. 00000020:00200000:1.0:1787826258.886377:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 434 items remaining. 00000020:00200000:1.0:1787826258.886379:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 433 items remaining. 00000020:00200000:1.0:1787826258.886380:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 432 items remaining. 00000020:00200000:1.0:1787826258.886382:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 431 items remaining. 00000020:00200000:1.0:1787826258.886383:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 430 items remaining. 00000020:00200000:1.0:1787826258.886384:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 429 items remaining. 00000020:00200000:1.0:1787826258.886386:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 428 items remaining. 00000020:00200000:1.0:1787826258.886389:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 427 items remaining. 00000020:00200000:1.0:1787826258.886390:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 426 items remaining. 00000020:00200000:1.0:1787826258.886392:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 425 items remaining. 00000020:00200000:1.0:1787826258.886394:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 424 items remaining. 00000020:00200000:1.0:1787826258.886395:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 423 items remaining. 00000020:00200000:1.0:1787826258.886397:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 422 items remaining. 00000020:00200000:1.0:1787826258.886398:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 421 items remaining. 00000020:00200000:1.0:1787826258.886400:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 420 items remaining. 00000020:00200000:1.0:1787826258.886401:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 419 items remaining. 00000020:00200000:1.0:1787826258.886402:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 418 items remaining. 00000020:00200000:1.0:1787826258.886404:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 417 items remaining. 00000020:00200000:1.0:1787826258.886406:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 416 items remaining. 00000020:00200000:1.0:1787826258.886407:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 415 items remaining. 00000020:00200000:1.0:1787826258.886409:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 414 items remaining. 00000020:00200000:1.0:1787826258.886411:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 413 items remaining. 00000020:00200000:1.0:1787826258.886412:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 412 items remaining. 00000020:00200000:1.0:1787826258.886414:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 411 items remaining. 00000020:00200000:1.0:1787826258.886415:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 410 items remaining. 00000020:00200000:1.0:1787826258.886416:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 409 items remaining. 00000020:00200000:1.0:1787826258.886418:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 408 items remaining. 00000020:00200000:1.0:1787826258.886419:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 407 items remaining. 00000020:00200000:1.0:1787826258.886421:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 406 items remaining. 00000020:00200000:1.0:1787826258.886422:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 405 items remaining. 00000020:00200000:1.0:1787826258.886424:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 404 items remaining. 00000020:00200000:1.0:1787826258.886426:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 403 items remaining. 00000020:00200000:1.0:1787826258.886427:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 402 items remaining. 00000020:00200000:1.0:1787826258.886429:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 401 items remaining. 00000020:00200000:1.0:1787826258.886430:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 400 items remaining. 00000020:00200000:1.0:1787826258.886431:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 399 items remaining. 00000020:00200000:1.0:1787826258.886433:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 398 items remaining. 00000020:00200000:1.0:1787826258.886434:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 397 items remaining. 00000020:00200000:1.0:1787826258.886436:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 396 items remaining. 00000020:00200000:1.0:1787826258.886438:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 395 items remaining. 00000020:00200000:1.0:1787826258.886440:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 394 items remaining. 00000020:00200000:1.0:1787826258.886441:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 393 items remaining. 00000020:00200000:1.0:1787826258.886443:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 392 items remaining. 00000020:00200000:1.0:1787826258.886444:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 391 items remaining. 00000020:00200000:1.0:1787826258.886445:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 390 items remaining. 00000020:00200000:1.0:1787826258.886460:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 389 items remaining. 00000020:00200000:1.0:1787826258.886462:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 388 items remaining. 00000020:00200000:1.0:1787826258.886464:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 387 items remaining. 00000020:00200000:1.0:1787826258.886465:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 386 items remaining. 00000020:00200000:1.0:1787826258.886467:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 385 items remaining. 00000020:00200000:1.0:1787826258.886468:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 384 items remaining. 00000020:00200000:1.0:1787826258.886470:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 383 items remaining. 00000020:00200000:1.0:1787826258.886471:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 382 items remaining. 00000020:00200000:1.0:1787826258.886473:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 381 items remaining. 00000020:00200000:1.0:1787826258.886474:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 380 items remaining. 00000020:00200000:1.0:1787826258.886475:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 379 items remaining. 00000020:00200000:1.0:1787826258.886477:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 378 items remaining. 00000020:00200000:1.0:1787826258.886479:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 377 items remaining. 00000020:00200000:1.0:1787826258.886481:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 376 items remaining. 00000020:00200000:1.0:1787826258.886483:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 375 items remaining. 00000020:00200000:1.0:1787826258.886484:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 374 items remaining. 00000020:00200000:1.0:1787826258.886486:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 373 items remaining. 00000020:00200000:1.0:1787826258.886488:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 372 items remaining. 00000020:00200000:1.0:1787826258.886489:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 371 items remaining. 00000020:00200000:1.0:1787826258.886491:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 370 items remaining. 00000020:00200000:1.0:1787826258.886493:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 369 items remaining. 00000020:00200000:1.0:1787826258.886494:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 368 items remaining. 00000020:00200000:1.0:1787826258.886496:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 367 items remaining. 00000020:00200000:1.0:1787826258.886497:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 366 items remaining. 00000020:00200000:1.0:1787826258.886499:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 365 items remaining. 00000020:00200000:1.0:1787826258.886501:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 364 items remaining. 00000020:00200000:1.0:1787826258.886503:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 363 items remaining. 00000020:00200000:1.0:1787826258.886505:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 362 items remaining. 00000020:00200000:1.0:1787826258.886507:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 361 items remaining. 00000020:00200000:1.0:1787826258.886509:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 360 items remaining. 00000020:00200000:1.0:1787826258.886511:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 359 items remaining. 00000020:00200000:1.0:1787826258.886512:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 358 items remaining. 00000020:00200000:1.0:1787826258.886513:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 357 items remaining. 00000020:00200000:1.0:1787826258.886514:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 356 items remaining. 00000020:00200000:1.0:1787826258.886516:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 355 items remaining. 00000020:00200000:1.0:1787826258.886517:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 354 items remaining. 00000020:00200000:1.0:1787826258.886519:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 353 items remaining. 00000020:00200000:1.0:1787826258.886520:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 352 items remaining. 00000020:00200000:1.0:1787826258.886522:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 351 items remaining. 00000020:00200000:1.0:1787826258.886523:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 350 items remaining. 00000020:00200000:1.0:1787826258.886526:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 349 items remaining. 00000020:00200000:1.0:1787826258.886527:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 348 items remaining. 00000020:00200000:1.0:1787826258.886529:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 347 items remaining. 00000020:00200000:1.0:1787826258.886531:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 346 items remaining. 00000020:00200000:1.0:1787826258.886533:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 345 items remaining. 00000020:00200000:1.0:1787826258.886535:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 344 items remaining. 00000020:00200000:1.0:1787826258.886537:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 343 items remaining. 00000020:00200000:1.0:1787826258.886539:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 342 items remaining. 00000020:00200000:1.0:1787826258.886541:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 341 items remaining. 00000020:00200000:1.0:1787826258.886542:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 340 items remaining. 00000020:00200000:1.0:1787826258.886544:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 339 items remaining. 00000020:00200000:1.0:1787826258.886546:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 338 items remaining. 00000020:00200000:1.0:1787826258.886547:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 337 items remaining. 00000020:00200000:1.0:1787826258.886549:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 336 items remaining. 00000020:00200000:1.0:1787826258.886550:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 335 items remaining. 00000020:00200000:1.0:1787826258.886551:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 334 items remaining. 00000020:00200000:1.0:1787826258.886553:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 333 items remaining. 00000020:00200000:1.0:1787826258.886554:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 332 items remaining. 00000020:00200000:1.0:1787826258.886557:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 331 items remaining. 00000020:00200000:1.0:1787826258.886558:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 330 items remaining. 00000020:00200000:1.0:1787826258.886559:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 329 items remaining. 00000020:00200000:1.0:1787826258.886561:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 328 items remaining. 00000020:00200000:1.0:1787826258.886563:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 327 items remaining. 00000020:00200000:1.0:1787826258.886565:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 326 items remaining. 00000020:00200000:1.0:1787826258.886567:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 325 items remaining. 00000020:00200000:1.0:1787826258.887392:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 324 items remaining. 00000020:00200000:1.0:1787826258.887400:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 323 items remaining. 00000020:00200000:1.0:1787826258.887402:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 322 items remaining. 00000020:00200000:1.0:1787826258.887403:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 321 items remaining. 00000020:00200000:1.0:1787826258.887405:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 320 items remaining. 00000020:00200000:1.0:1787826258.887406:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 319 items remaining. 00000020:00200000:1.0:1787826258.887408:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 318 items remaining. 00000020:00200000:1.0:1787826258.887409:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 317 items remaining. 00000020:00200000:1.0:1787826258.887410:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 316 items remaining. 00000020:00200000:1.0:1787826258.887412:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 315 items remaining. 00000020:00200000:1.0:1787826258.887413:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 314 items remaining. 00000020:00200000:1.0:1787826258.887415:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 313 items remaining. 00000020:00200000:1.0:1787826258.887416:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 312 items remaining. 00000020:00200000:1.0:1787826258.887418:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 311 items remaining. 00000020:00200000:1.0:1787826258.887419:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 310 items remaining. 00000020:00200000:1.0:1787826258.887421:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 309 items remaining. 00000020:00200000:1.0:1787826258.887423:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 308 items remaining. 00000020:00200000:1.0:1787826258.887424:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 307 items remaining. 00000020:00200000:1.0:1787826258.887425:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 306 items remaining. 00000020:00200000:1.0:1787826258.887427:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 305 items remaining. 00000020:00200000:1.0:1787826258.887428:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 304 items remaining. 00000020:00200000:1.0:1787826258.887430:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 303 items remaining. 00000020:00200000:1.0:1787826258.887431:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 302 items remaining. 00000020:00200000:1.0:1787826258.887432:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 301 items remaining. 00000020:00200000:1.0:1787826258.887434:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 300 items remaining. 00000020:00200000:1.0:1787826258.887436:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 299 items remaining. 00000020:00200000:1.0:1787826258.887438:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 298 items remaining. 00000020:00200000:1.0:1787826258.887439:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 297 items remaining. 00000020:00200000:1.0:1787826258.887441:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 296 items remaining. 00000020:00200000:1.0:1787826258.887443:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 295 items remaining. 00000020:00200000:1.0:1787826258.887444:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 294 items remaining. 00000020:00200000:1.0:1787826258.887446:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 293 items remaining. 00000020:00200000:1.0:1787826258.887447:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 292 items remaining. 00000020:00200000:1.0:1787826258.887449:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 291 items remaining. 00000020:00200000:1.0:1787826258.887450:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 290 items remaining. 00000020:00200000:1.0:1787826258.887452:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 289 items remaining. 00000020:00200000:1.0:1787826258.887453:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 288 items remaining. 00000020:00200000:1.0:1787826258.887454:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 287 items remaining. 00000020:00200000:1.0:1787826258.887455:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 286 items remaining. 00000020:00200000:1.0:1787826258.887457:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 285 items remaining. 00000020:00200000:1.0:1787826258.887459:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 284 items remaining. 00000020:00200000:1.0:1787826258.887460:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 283 items remaining. 00000020:00200000:1.0:1787826258.887462:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 282 items remaining. 00000020:00200000:1.0:1787826258.887463:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 281 items remaining. 00000020:00200000:1.0:1787826258.887465:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 280 items remaining. 00000020:00200000:1.0:1787826258.887466:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 279 items remaining. 00000020:00200000:1.0:1787826258.887467:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 278 items remaining. 00000020:00200000:1.0:1787826258.887469:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 277 items remaining. 00000020:00200000:1.0:1787826258.887470:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 276 items remaining. 00000020:00200000:1.0:1787826258.887472:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 275 items remaining. 00000020:00200000:1.0:1787826258.887473:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 274 items remaining. 00000020:00200000:1.0:1787826258.887475:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 273 items remaining. 00000020:00200000:1.0:1787826258.887476:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 272 items remaining. 00000020:00200000:1.0:1787826258.887478:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 271 items remaining. 00000020:00200000:1.0:1787826258.887479:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 270 items remaining. 00000020:00200000:1.0:1787826258.887481:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 269 items remaining. 00000020:00200000:1.0:1787826258.887482:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 268 items remaining. 00000020:00200000:1.0:1787826258.887484:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 267 items remaining. 00000020:00200000:1.0:1787826258.887485:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 266 items remaining. 00000020:00200000:1.0:1787826258.887487:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 265 items remaining. 00000020:00200000:1.0:1787826258.887488:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 264 items remaining. 00000020:00200000:1.0:1787826258.887489:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 263 items remaining. 00000020:00200000:1.0:1787826258.887491:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 262 items remaining. 00000020:00200000:1.0:1787826258.887492:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 261 items remaining. 00000020:00200000:1.0:1787826258.887493:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 260 items remaining. 00000020:00200000:1.0:1787826258.887494:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 259 items remaining. 00000020:00200000:1.0:1787826258.887496:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 258 items remaining. 00000020:00200000:1.0:1787826258.887498:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 257 items remaining. 00000020:00200000:1.0:1787826258.887499:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000020:00200000:1.0:1787826258.887501:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:1.0:1787826258.887502:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:1.0:1787826258.887504:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:1.0:1787826258.887505:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:1.0:1787826258.887506:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:1.0:1787826258.887508:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:1.0:1787826258.887509:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:1.0:1787826258.887511:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:1.0:1787826258.887512:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:1.0:1787826258.887514:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:1.0:1787826258.887515:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:1.0:1787826258.887517:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:1.0:1787826258.887518:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:1.0:1787826258.887520:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:1.0:1787826258.887521:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:1.0:1787826258.887523:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:1.0:1787826258.887524:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:1.0:1787826258.887526:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:1.0:1787826258.887528:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:1.0:1787826258.887529:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:1.0:1787826258.887531:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:1.0:1787826258.887532:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:1.0:1787826258.887534:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:1.0:1787826258.887535:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:1.0:1787826258.887537:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:1.0:1787826258.887539:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:1.0:1787826258.887540:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:1.0:1787826258.887542:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:1.0:1787826258.887543:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:1.0:1787826258.887544:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:1.0:1787826258.887546:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:1.0:1787826258.887547:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:1.0:1787826258.887549:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:1.0:1787826258.887550:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:1.0:1787826258.887552:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:1.0:1787826258.887554:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:1.0:1787826258.887555:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:1.0:1787826258.887557:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:1.0:1787826258.887558:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:1.0:1787826258.887560:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:1.0:1787826258.887562:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:1.0:1787826258.887563:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:1.0:1787826258.887565:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:1.0:1787826258.887566:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:1.0:1787826258.887567:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:1.0:1787826258.887581:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:1.0:1787826258.887582:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:1.0:1787826258.887584:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:1.0:1787826258.887586:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:1.0:1787826258.887588:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:1.0:1787826258.887589:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:1.0:1787826258.887591:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:1.0:1787826258.887593:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:1.0:1787826258.887595:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:1.0:1787826258.887596:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:1.0:1787826258.887597:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:1.0:1787826258.887599:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:1.0:1787826258.887600:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:1.0:1787826258.887602:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:1.0:1787826258.887603:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:1.0:1787826258.887604:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:1.0:1787826258.887605:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:1.0:1787826258.887631:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:1.0:1787826258.887633:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:1.0:1787826258.887635:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:1.0:1787826258.887637:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:1.0:1787826258.887638:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:1.0:1787826258.887641:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:1.0:1787826258.887643:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:1.0:1787826258.887645:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:1.0:1787826258.887646:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:1.0:1787826258.887649:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:1.0:1787826258.887650:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:1.0:1787826258.887652:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:1.0:1787826258.887653:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:1.0:1787826258.887655:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:1.0:1787826258.887656:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:1.0:1787826258.887657:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:1.0:1787826258.887659:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:1.0:1787826258.887660:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:1.0:1787826258.887661:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:1.0:1787826258.887663:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:1.0:1787826258.887665:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:1.0:1787826258.887666:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:1.0:1787826258.887668:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:1.0:1787826258.887670:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:1.0:1787826258.887671:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:1.0:1787826258.887673:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:1.0:1787826258.887675:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:1.0:1787826258.887677:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:1.0:1787826258.887679:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:1.0:1787826258.887680:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:1.0:1787826258.887682:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:1.0:1787826258.887683:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:1.0:1787826258.887685:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:1.0:1787826258.887686:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:1.0:1787826258.887687:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:1.0:1787826258.887689:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:1.0:1787826258.887691:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:1.0:1787826258.887692:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:1.0:1787826258.887694:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:1.0:1787826258.887696:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:1.0:1787826258.887697:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:1.0:1787826258.887699:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:1.0:1787826258.887701:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:1.0:1787826258.887702:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:1.0:1787826258.887704:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:1.0:1787826258.887706:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:1.0:1787826258.887708:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:1.0:1787826258.887709:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:1.0:1787826258.887710:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:1.0:1787826258.887712:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:1.0:1787826258.887714:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:1.0:1787826258.887715:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:1.0:1787826258.887717:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:1.0:1787826258.887718:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:1.0:1787826258.887720:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:1.0:1787826258.887722:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:1.0:1787826258.887723:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:1.0:1787826258.887726:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:1.0:1787826258.887727:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:1.0:1787826258.887729:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:1.0:1787826258.887730:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:1.0:1787826258.887732:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:1.0:1787826258.887733:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:1.0:1787826258.887735:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:1.0:1787826258.887737:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:1.0:1787826258.887739:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:1.0:1787826258.887741:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:1.0:1787826258.887743:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:1.0:1787826258.887744:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:1.0:1787826258.887746:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:1.0:1787826258.887747:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:1.0:1787826258.887749:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:1.0:1787826258.887751:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:1.0:1787826258.887752:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:1.0:1787826258.887754:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:1.0:1787826258.887756:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:1.0:1787826258.887758:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:1.0:1787826258.887759:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:1.0:1787826258.887761:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:1.0:1787826258.887763:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:1.0:1787826258.887764:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:1.0:1787826258.887766:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:1.0:1787826258.887768:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:1.0:1787826258.887769:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:1.0:1787826258.887771:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:1.0:1787826258.887772:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:1.0:1787826258.887774:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:1.0:1787826258.887776:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:1.0:1787826258.889049:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:1.0:1787826258.889054:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:1.0:1787826258.889055:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:1.0:1787826258.889057:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:1.0:1787826258.889059:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:1.0:1787826258.889060:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:1.0:1787826258.889062:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:1.0:1787826258.889064:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:1.0:1787826258.889066:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:1.0:1787826258.889068:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:1.0:1787826258.889070:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:1.0:1787826258.889072:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:1.0:1787826258.889074:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:1.0:1787826258.889076:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:1.0:1787826258.889078:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:1.0:1787826258.889080:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:1.0:1787826258.889081:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:1.0:1787826258.889083:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:1.0:1787826258.889085:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:1.0:1787826258.889087:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:1.0:1787826258.889088:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:1.0:1787826258.889090:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:1.0:1787826258.889091:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:1.0:1787826258.889093:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:1.0:1787826258.889094:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:1.0:1787826258.889096:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:1.0:1787826258.889098:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:1.0:1787826258.889099:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:1.0:1787826258.889101:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:1.0:1787826258.889103:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:1.0:1787826258.889106:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:1.0:1787826258.889107:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:1.0:1787826258.889109:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:1.0:1787826258.889111:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:1.0:1787826258.889113:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:1.0:1787826258.889114:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:1.0:1787826258.889116:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:1.0:1787826258.889118:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:1.0:1787826258.889119:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:1.0:1787826258.889121:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:1.0:1787826258.889123:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:1.0:1787826258.889125:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:1.0:1787826258.889127:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:1.0:1787826258.889129:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:1.0:1787826258.889131:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:1.0:1787826258.889133:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:1.0:1787826258.889134:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:1.0:1787826258.889136:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:1.0:1787826258.889138:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:1.0:1787826258.889139:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:1.0:1787826258.889141:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:1.0:1787826258.889142:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:1.0:1787826258.889144:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:1.0:1787826258.889145:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:1.0:1787826258.889147:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:1.0:1787826258.889149:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:1.0:1787826258.889150:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:1.0:1787826258.889152:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:1.0:1787826258.889154:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:1.0:1787826258.889156:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:1.0:1787826258.889158:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:1.0:1787826258.889160:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:1.0:1787826258.889162:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:1.0:1787826258.889164:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:1.0:1787826258.889165:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:1.0:1787826258.889167:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:1.0:1787826258.889169:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:1.0:1787826258.889171:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:1.0:1787826258.889173:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:1.0:1787826258.889175:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:1.0:1787826258.889177:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:1.0:1787826258.889178:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:1.0:1787826258.889180:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:1.0:1787826258.889182:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:1.0:1787826258.889184:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:1.0:1787826258.889186:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:1.0:1787826258.889188:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:1.0:1787826258.889190:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:1.0:1787826258.889191:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:1.0:1787826258.889193:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:1.0:1787826258.889195:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:1.0:1787826258.889197:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:1.0:1787826258.889198:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:1.0:1787826258.889200:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:1.0:1787826258.889202:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:1.0:1787826258.889203:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:1.0:1787826258.889205:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:1.0:1787826258.889207:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:1.0:1787826258.889208:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:1.0:1787826258.889210:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:1.0:1787826258.889212:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:1.0:1787826258.889214:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:1.0:1787826258.889216:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:1.0:1787826258.889217:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:1.0:1787826258.889220:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:1.0:1787826258.889221:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:1.0:1787826258.889223:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:1.0:1787826258.889225:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:1.0:1787826258.889226:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:1.0:1787826258.889228:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:1.0:1787826258.889229:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:1.0:1787826258.889231:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:1.0:1787826258.889232:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:1.0:1787826258.889234:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:1.0:1787826258.889236:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:1.0:1787826258.889237:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1787826258.889482:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1787826258.889504:0:2348:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907d908aa0c0 00000100:00100000:1.0:1787826258.889533:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011394560:192.168.203.104@tcp:4:dd.0 00000080:00200000:2.0:1787826258.889550:0:65896:0:(file.c:2207:ll_file_io_generic()) Range unlock [0, 511] 00000080:00200000:2.0:1787826258.889567:0:65896:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 6 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826258.889584:0:65896:0:(file.c:2240:ll_file_io_generic()) f204a.sanity-flr: 2 io complete with rc: 0, result: 2097152, restart: 0 00000080:00200000:2.0:1787826258.889600:0:65896:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000020:00200000:2.0:1787826258.889604:0:65896:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826258.889634:0:65896:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 2097152 00000080:00200008:2.0:1787826258.889670:0:65896:0:(file.c:479:ll_file_release()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0](ffff907d92219148), flags 100001 00000002:00100000:2.0:1787826258.890024:0:65896:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d908a2580 x1874670011394048/t8589942938(8589942938) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1088/1160 e 0 to 0 dl 1787826276 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1787826258.890036:0:65896:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1787826258.890041:0:65896:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114a940 x1874670011394816/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1787826258.890062:0:65896:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:65896:1874670011394816:192.168.203.104@tcp:35:dd.0 00000100:00100000:2.0:1787826258.890219:0:65896:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6b40 going to sleep for 11 seconds 00000100:00100000:2.0:1787826258.894878:0:65896:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:65896:1874670011394816:192.168.203.104@tcp:35:dd.0 00000002:00100000:2.0:1787826258.894921:0:65896:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d908a2580 x1874670011394048/t8589942938(8589942938) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1088/1160 e 0 to 0 dl 1787826276 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826258.909965:0:65897:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826258.909994:0:65897:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826258.910013:0:65897:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826258.910029:0:65897:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1787826258.910038:0:65897:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826258.910045:0:65897:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826258.910049:0:65897:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:1.0:1787826258.910056:0:65897:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd14195bbf8 00010000:00010000:1.0:1787826258.910060:0:65897:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826258.910073:0:65897:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1787826258.910099:0:65897:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826258.910107:0:65897:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=27 00000080:00200000:1.0:1787826258.910112:0:65897:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 1 00000080:00200000:1.0:1787826258.910122:0:65897:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), file ffff907d99b3a780, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1787826258.910129:0:65897:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), intent=open 00000002:00010000:1.0:1787826258.910141:0:65897:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204a.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1fc:0x0], intent: open flags 0304001 00010000:00010000:1.0:1787826258.910199:0:65897:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b697 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.910207:0:65897:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b697 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826258.910218:0:65897:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826258.910237:0:65897:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a1e00 x1874670011395072/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826258.910247:0:65897:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b697 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826258.910269:0:65897:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65897:1874670011395072:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826258.910436:0:65897:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb600 going to sleep for 13 seconds 00000100:00100000:1.0:1787826258.917717:0:65897:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65897:1874670011395072:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826258.917755:0:65897:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b697 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.917773:0:65897:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b697 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.917782:0:65897:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b697 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.917789:0:65897:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b697 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.917798:0:65897:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b697 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.917805:0:65897:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0be00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826258.917809:0:65897:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b697 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.917817:0:65897:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b697 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.917824:0:65897:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b697 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.917831:0:65897:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826258.917837:0:65897:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b697 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.917844:0:65897:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b697 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826258.917862:0:65897:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908a1e00 x1874670011395072/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826276 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826258.917876:0:65897:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1fc:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826258.918011:0:65897:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826258.918028:0:65897:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826258.918042:0:65897:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1787826258.918054:0:65897:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1787826258.918061:0:65897:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826258.918065:0:65897:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826258.918069:0:65897:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:1.0:1787826258.918420:0:65897:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd14195bbf8 00010000:00010000:1.0:1787826258.918427:0:65897:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1787826258.918436:0:65897:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1787826258.918451:0:65897:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826258.918459:0:65897:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=267 00000080:00200000:1.0:1787826258.918466:0:65897:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 44 00000080:00200008:1.0:1787826258.918474:0:65897:0:(file.c:1090:ll_file_open()) START file d204a.sanity-flr:[0x200000404:0x1fc:0x0](ffff907d845442c8), flags 304000 00000002:00010000:1.0:1787826258.918490:0:65897:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1fc:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:1.0:1787826258.918534:0:65897:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b69e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.918545:0:65897:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b69e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826258.918555:0:65897:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826258.918562:0:65897:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a1e00 x1874670011395328/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826258.918582:0:65897:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b69e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826258.918601:0:65897:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65897:1874670011395328:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826258.918969:0:65897:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb600 going to sleep for 13 seconds 00000100:00100000:1.0:1787826258.924243:0:65897:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65897:1874670011395328:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826258.924265:0:65897:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b69e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.924278:0:65897:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b69e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.924284:0:65897:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b69e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.924291:0:65897:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b69e lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.924299:0:65897:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b69e lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.924305:0:65897:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0ac00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826258.924310:0:65897:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b69e lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.924316:0:65897:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b69e lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.924323:0:65897:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b69e lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.924330:0:65897:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826258.924337:0:65897:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b69e lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.924345:0:65897:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b69e lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65897 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826258.924373:0:65897:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffff907d908a1e00 x1874670011395328/t8589942941(8589942941) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826276 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826258.924387:0:65897:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a1e00 x1874670011395328/t8589942941(8589942941) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826276 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826258.924400:0:65897:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826258.924419:0:65897:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x1fc:0x0], updating i_size 4096 00000002:00100000:1.0:1787826258.924437:0:65897:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a1e00 x1874670011395328/t8589942941(8589942941) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826276 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1787826258.924522:0:65897:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8) cmd=c0086915 arg=21323d0 00000100:00100000:1.0:1787826258.924592:0:65897:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65897:1874670011395456:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826258.924690:0:65897:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb600 going to sleep for 13 seconds 00000100:00100000:1.0:1787826258.928053:0:65897:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65897:1874670011395456:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826258.930567:0:65897:0:(file.c:479:ll_file_release()) START file d204a.sanity-flr:[0x200000404:0x1fc:0x0](ffff907d845442c8), flags 304000 00000002:00100000:1.0:1787826258.930933:0:65897:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d908a1e00 x1874670011395328/t8589942941(8589942941) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826276 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826258.930946:0:65897:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826258.930953:0:65897:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a3480 x1874670011395712/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826258.930977:0:65897:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65897:1874670011395712:192.168.203.104@tcp:35:lfs.0 00000100:00100000:1.0:1787826258.931171:0:65897:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb600 going to sleep for 11 seconds 00000100:00100000:1.0:1787826258.933988:0:65897:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65897:1874670011395712:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826258.934018:0:65897:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d908a1e00 x1874670011395328/t8589942941(8589942941) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826276 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1787826258.955389:0:65899:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1787826258.955400:0:65899:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826258.955419:0:65899:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826258.955438:0:65899:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1787826258.955452:0:65899:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1787826258.955460:0:65899:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826258.955467:0:65899:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826258.955471:0:65899:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:2.0:1787826258.955478:0:65899:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141de3cb8 00010000:00010000:2.0:1787826258.955482:0:65899:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1787826258.955491:0:65899:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:2.0:1787826258.955862:0:65899:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826258.955871:0:65899:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826258.955879:0:65899:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1787826258.955889:0:65899:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:2.0:1787826258.955896:0:65899:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826258.955900:0:65899:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826258.955902:0:65899:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:2.0:1787826258.955907:0:65899:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141de3bb0 00010000:00010000:2.0:1787826258.955909:0:65899:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1787826258.955917:0:65899:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:2.0:1787826258.955927:0:65899:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:2.0:1787826258.955935:0:65899:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:2.0:1787826258.955952:0:65899:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffd95b7e2dc 00000080:00200008:2.0:1787826258.955963:0:65899:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:3.0:1787826258.957698:0:65904:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826258.957721:0:65904:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826258.957737:0:65904:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:3.0:1787826258.957752:0:65904:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:3.0:1787826258.957760:0:65904:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826258.957768:0:65904:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826258.957771:0:65904:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:3.0:1787826258.957778:0:65904:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141c27bf8 00010000:00010000:3.0:1787826258.957782:0:65904:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:3.0:1787826258.957790:0:65904:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:3.0:1787826258.957804:0:65904:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826258.957810:0:65904:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=27 00000080:00200000:3.0:1787826258.957815:0:65904:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 1 00000080:00200000:3.0:1787826258.957827:0:65904:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), file ffff907d98b6ca80, open_flags 18800, mode 0 opened 0 00000080:00200000:3.0:1787826258.957833:0:65904:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), intent=open 00000002:00010000:3.0:1787826258.957846:0:65904:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204a.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1fc:0x0], intent: open flags 0304001 00010000:00010000:3.0:1787826258.957896:0:65904:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b6a5 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65904 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826258.957904:0:65904:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b6a5 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65904 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826258.957914:0:65904:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826258.957925:0:65904:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83527840 x1874670011395968/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826258.957935:0:65904:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b6a5 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65904 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826258.957954:0:65904:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65904:1874670011395968:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826258.958845:0:65904:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d960579c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826258.961203:0:65904:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65904:1874670011395968:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826258.961226:0:65904:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b6a5 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65904 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.961239:0:65904:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b6a5 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65904 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.961245:0:65904:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b6a5 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65904 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.961252:0:65904:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b6a5 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65904 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.961261:0:65904:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b6a5 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65904 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.961272:0:65904:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d86042200 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826258.961277:0:65904:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b6a5 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 65904 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.961283:0:65904:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b6a5 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 65904 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.961290:0:65904:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b6a5 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 65904 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.961296:0:65904:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826258.961303:0:65904:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b6a5 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65904 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.961310:0:65904:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b6a5 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65904 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826258.961328:0:65904:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d83527840 x1874670011395968/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826276 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826258.961340:0:65904:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1fc:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826258.961563:0:65904:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826258.961593:0:65904:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826258.961627:0:65904:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1787826258.961639:0:65904:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1787826258.961647:0:65904:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826258.961653:0:65904:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826258.961656:0:65904:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:1.0:1787826258.961662:0:65904:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141c27bf8 00010000:00010000:1.0:1787826258.961666:0:65904:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1787826258.961674:0:65904:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1787826258.961688:0:65904:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826258.961695:0:65904:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=267 00000080:00200000:1.0:1787826258.961702:0:65904:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 44 00000080:00200008:1.0:1787826258.961710:0:65904:0:(file.c:1090:ll_file_open()) START file d204a.sanity-flr:[0x200000404:0x1fc:0x0](ffff907d845442c8), flags 304000 00000002:00010000:1.0:1787826258.961736:0:65904:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1fc:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240304001 00010000:00010000:1.0:1787826258.961777:0:65904:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b6ac lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65904 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.961784:0:65904:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b6ac lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65904 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826258.961795:0:65904:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826258.961801:0:65904:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a3480 x1874670011396224/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826258.961812:0:65904:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b6ac lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65904 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826258.961832:0:65904:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65904:1874670011396224:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826258.961994:0:65904:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb600 going to sleep for 13 seconds 00000100:00100000:1.0:1787826258.964682:0:65904:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65904:1874670011396224:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826258.964712:0:65904:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b6ac lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f77a expref: -99 pid: 65904 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.964723:0:65904:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b6ac lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f77a expref: -99 pid: 65904 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.964734:0:65904:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b6ac lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f77a expref: -99 pid: 65904 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.964745:0:65904:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b6ac lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f77a expref: -99 pid: 65904 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826258.964759:0:65904:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x200002b, status=0 req@ffff907d908a3480 x1874670011396224/t8589942943(8589942943) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826276 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826258.964772:0:65904:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a3480 x1874670011396224/t8589942943(8589942943) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826276 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826258.964783:0:65904:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b6ac lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f77a expref: -99 pid: 65904 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.964793:0:65904:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/508 (0 0) 00000080:00200000:1.0:1787826258.964799:0:65904:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826258.964816:0:65904:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x1fc:0x0], updating i_size 4096 00000080:00010000:1.0:1787826258.964821:0:65904:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fc:0x0](ffff907d845442c8) for lock 0xe5f736940357b6ac 00000080:00010000:1.0:1787826258.964828:0:65904:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b6ac from it ffff9dd141c27bd0 00010000:00010000:1.0:1787826258.964832:0:65904:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b6ac lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f77a expref: -99 pid: 65904 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826258.964846:0:65904:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b6ac lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f77a expref: -99 pid: 65904 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1787826258.964870:0:65904:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a3480 x1874670011396224/t8589942943(8589942943) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826276 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1787826258.964949:0:65904:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8) cmd=c1406916 arg=14a9a20 00000100:00100000:1.0:1787826258.965005:0:65904:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65904:1874670011396352:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826258.965076:0:65904:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb600 going to sleep for 13 seconds 00000100:00100000:1.0:1787826258.975719:0:65904:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65904:1874670011396352:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826258.975887:0:65904:0:(file.c:479:ll_file_release()) START file d204a.sanity-flr:[0x200000404:0x1fc:0x0](ffff907d845442c8), flags 304000 00010000:00010000:1.0:1787826258.975915:0:65904:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b6ac lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f77a expref: -99 pid: 65904 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200008:1.0:1787826258.996336:0:65906:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1787826258.996347:0:65906:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826258.996367:0:65906:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826258.996386:0:65906:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1787826258.996400:0:65906:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1787826258.996409:0:65906:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826258.996416:0:65906:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826258.996419:0:65906:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:1.0:1787826258.996428:0:65906:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141de3cb8 00010000:00010000:1.0:1787826258.996431:0:65906:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1787826258.996441:0:65906:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1787826258.996541:0:65906:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826258.996549:0:65906:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826258.996556:0:65906:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1787826258.996565:0:65906:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1787826258.996583:0:65906:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826258.996587:0:65906:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826258.996590:0:65906:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:1.0:1787826258.996593:0:65906:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141de3bb0 00010000:00010000:1.0:1787826258.996596:0:65906:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1787826258.996605:0:65906:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:1.0:1787826258.996678:0:65906:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:1.0:1787826258.996685:0:65906:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:1.0:1787826258.996702:0:65906:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffe40a5d95c 00000080:00200008:1.0:1787826258.996714:0:65906:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:2.0:1787826258.998200:0:65908:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826258.998217:0:65908:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826258.998230:0:65908:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1787826258.998246:0:65908:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:2.0:1787826258.998254:0:65908:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826258.998260:0:65908:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826258.998263:0:65908:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:2.0:1787826258.998269:0:65908:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd14195bbf8 00010000:00010000:2.0:1787826258.998273:0:65908:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1787826258.998280:0:65908:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:2.0:1787826258.998292:0:65908:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826258.998298:0:65908:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=27 00000080:00200000:2.0:1787826258.998304:0:65908:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 1 00000080:00200000:2.0:1787826258.998314:0:65908:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), file ffff907d83b1ef00, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1787826258.998320:0:65908:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), intent=open 00000002:00010000:2.0:1787826258.998332:0:65908:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204a.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1fc:0x0], intent: open flags 0304001 00010000:00010000:2.0:1787826258.998383:0:65908:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6b3 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 65908 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826258.998391:0:65908:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6b3 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65908 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826258.998401:0:65908:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826258.998412:0:65908:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114a940 x1874670011396608/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826258.998423:0:65908:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6b3 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65908 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826258.998443:0:65908:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65908:1874670011396608:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826258.998745:0:65908:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6b40 going to sleep for 13 seconds 00000100:00100000:2.0:1787826259.001681:0:65908:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65908:1874670011396608:192.168.203.104@tcp:101:lfs.0 00010000:00010000:2.0:1787826259.001701:0:65908:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6b3 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65908 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.001714:0:65908:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6b3 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 65908 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.001720:0:65908:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6b3 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 65908 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.001738:0:65908:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6b3 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 65908 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.001747:0:65908:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6b3 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 65908 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.001753:0:65908:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907dc2177600 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:2.0:1787826259.001758:0:65908:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6b3 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 65908 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.001765:0:65908:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6b3 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 65908 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.001773:0:65908:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6b3 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 65908 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.001781:0:65908:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1787826259.001787:0:65908:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6b3 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 65908 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.001794:0:65908:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6b3 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 65908 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826259.001812:0:65908:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d9114a940 x1874670011396608/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826276 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1787826259.001826:0:65908:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1fc:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1787826259.002021:0:65908:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826259.002038:0:65908:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826259.002052:0:65908:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1787826259.002063:0:65908:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:2.0:1787826259.002070:0:65908:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826259.002075:0:65908:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826259.002420:0:65908:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:2.0:1787826259.002428:0:65908:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd14195bbf8 00010000:00010000:2.0:1787826259.002432:0:65908:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1787826259.002441:0:65908:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:2.0:1787826259.002462:0:65908:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826259.002469:0:65908:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=267 00000080:00200000:2.0:1787826259.002475:0:65908:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 44 00000080:00200008:2.0:1787826259.002483:0:65908:0:(file.c:1090:ll_file_open()) START file d204a.sanity-flr:[0x200000404:0x1fc:0x0](ffff907d845442c8), flags 304000 00000080:00200080:2.0:1787826259.002516:0:65908:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8) cmd=c1406916 arg=15cda20 00000100:00100000:2.0:1787826259.002579:0:65908:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65908:1874670011396736:192.168.203.104@tcp:34:lfs.0 00000100:00100000:2.0:1787826259.004190:0:65908:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6b40 going to sleep for 12 seconds 00000100:00100000:2.0:1787826259.011398:0:65908:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65908:1874670011396736:192.168.203.104@tcp:34:lfs.0 00000080:00200008:2.0:1787826259.011549:0:65908:0:(file.c:479:ll_file_release()) START file d204a.sanity-flr:[0x200000404:0x1fc:0x0](ffff907d845442c8), flags 304000 00010000:00010000:2.0:1787826259.011586:0:65908:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b6ac lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f77a expref: -99 pid: 65904 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1787826259.026830:0:65912:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826259.026859:0:65912:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826259.026878:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:2.0:1787826259.026892:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:2.0:1787826259.026899:0:65912:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826259.026906:0:65912:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826259.026909:0:65912:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:2.0:1787826259.026916:0:65912:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd14195bb58 00010000:00010000:2.0:1787826259.026919:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:2.0:1787826259.026928:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:2.0:1787826259.026941:0:65912:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826259.026948:0:65912:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=25 00000080:00200000:2.0:1787826259.026954:0:65912:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 1 00000080:00200000:2.0:1787826259.026964:0:65912:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), flags=9 00000080:00200000:2.0:1787826259.026968:0:65912:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), intent=getattr 00000002:00010000:2.0:1787826259.026978:0:65912:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204a.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1fc:0x0], intent: getattr flags 01 00010000:00010000:2.0:1787826259.027208:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 65912 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.027217:0:65912:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65912 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.027227:0:65912:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65912 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826259.027253:0:65912:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011396864:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826259.027452:0:65912:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6b40 going to sleep for 13 seconds 00000100:00100000:2.0:1787826259.031580:0:65912:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011396864:192.168.203.104@tcp:101:lfs.0 00010000:00010000:2.0:1787826259.031605:0:65912:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.031869:0:65912:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.031884:0:65912:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.031897:0:65912:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.031908:0:65912:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.031921:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.031930:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.031939:0:65912:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826259.031951:0:65912:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d9114a940 x1874670011396864/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/1136 e 0 to 0 dl 1787826277 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1787826259.031963:0:65912:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1787826259.032033:0:65912:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.032050:0:65912:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/509 (0 0) 00000080:00200000:2.0:1787826259.032056:0:65912:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1fc:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1787826259.032231:0:65912:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x1fd:0x0](ffff907d92219148): rc = 0 00000080:00200000:2.0:1787826259.032250:0:65912:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1fd:0x0]: layout version change: 6 -> 6 00000080:00010000:2.0:1787826259.032255:0:65912:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) for lock 0xe5f736940357b6ba 00000080:00010000:2.0:1787826259.032265:0:65912:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) 00000080:00010000:2.0:1787826259.032269:0:65912:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b6ba from it ffff9dd14195bb20 00010000:00010000:2.0:1787826259.032273:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.032282:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1787826259.032320:0:65912:0:(file.c:6313:ll_getattr_dentry()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0](ffff907d92219148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1787826259.032327:0:65912:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148),name=f204a.sanity-flr 00000002:00010000:2.0:1787826259.032344:0:65912:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1fd:0x0]) in obj [0x200000404:0x1fc:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826259.032355:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826259.032430:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826259.032438:0:65912:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1fc:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826259.032444:0:65912:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) 00000080:00010000:2.0:1787826259.032447:0:65912:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) for lock 0xe5f736940357b6ba 00000080:00010000:2.0:1787826259.032452:0:65912:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b6ba from it ffff9dd14195bcb8 00010000:00010000:2.0:1787826259.032455:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826259.032461:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1787826259.032479:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/1, layout version 6 restore needed 0 00020000:00200000:2.0:1787826259.032490:0:65912:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826259.032493:0:65912:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1787826259.032496:0:65912:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1fd:0x0] 00000008:00010000:2.0:1787826259.032513:0:65912:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826259.032516:0:65912:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d963221b0, osc lock ffff907d830a2c30, flags 0x400000400001002 00010000:00010000:2.0:1787826259.032558:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.032579:0:65912:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.032591:0:65912:0:(ldlm_request.c:1139:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff907d9114a940 x1874670011396992/t0(0) o101->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1787826259.032688:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011396992:192.168.203.104@tcp:101:lfs.0 00000020:00200000:2.0:1787826259.032788:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826259.035600:0:2350:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.035661:0:2350:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f7e3 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.035677:0:2350:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f7e3 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.035689:0:2350:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f7e3 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826259.035704:0:2350:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f7e3 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:2.0:1787826259.035717:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:2.0:1787826259.035735:0:2350:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f7e3 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.035745:0:2350:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f7e3 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.035754:0:2350:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f7e3 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.035763:0:2350:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f7e3 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:2.0:1787826259.035782:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011396992:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826259.040255:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b6c8 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x7918213f4fc0f7f1 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.040263:0:65912:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b6c8 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x7918213f4fc0f7f1 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.040289:0:65912:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:0.0:1787826259.040296:0:65912:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1fc:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1787826259.040354:0:65912:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1fd:0x0]: layout version change: 6 -> 6 00000080:00010000:0.0:1787826259.040369:0:65912:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) for lock 0xe5f736940357b6ba 00000080:00010000:0.0:1787826259.040377:0:65912:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b6ba from it ffff9dd14195bbf0 00010000:00010000:0.0:1787826259.040380:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826259.040390:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000002:00100000:0.0:1787826259.040415:0:65912:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90846d00 x1874670011397248/t8589942944(8589942944) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826277 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826259.040579:0:65912:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148), inode mode 81a4 mask 4 00000080:00200000:0.0:1787826259.040588:0:65912:0:(xattr.c:760:ll_xattr_get()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148), xattr lov 00000080:00200008:0.0:1787826259.040657:0:65912:0:(file.c:6313:ll_getattr_dentry()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0](ffff907d92219148), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1787826259.040665:0:65912:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148),name=f204a.sanity-flr 00000002:00010000:0.0:1787826259.040684:0:65912:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1fd:0x0]) in obj [0x200000404:0x1fc:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826259.040693:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826259.040703:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1787826259.040711:0:65912:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1fc:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826259.040717:0:65912:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) 00000080:00010000:0.0:1787826259.040720:0:65912:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) for lock 0xe5f736940357b6ba 00010000:00010000:1.0:1787826259.040876:0:60483:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826259.040884:0:60483:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b6c8 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7918213f4fc0f7f1 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.040891:0:60483:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883ba00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826259.040895:0:60483:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b6c8 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7918213f4fc0f7f1 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.040903:0:60483:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b6c8 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7918213f4fc0f7f1 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.040910:0:60483:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b6c8 lrc: 3/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc0f7f1 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.040924:0:60483:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826259.040961:0:60483:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b6c8 lrc: 2/0,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0f7f1 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.040966:0:60483:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826259.040989:0:60483:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b6c8 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0f7f1 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.040996:0:60483:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b6c8 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0f7f1 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.041010:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826259.041012:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826259.041014:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826259.041031:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011397376:192.168.203.104@tcp:103:ldlm_bl.0 00000080:00010000:0.0:1787826259.042227:0:65912:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b6ba from it ffff9dd14195bcd8 00010000:00010000:0.0:1787826259.042237:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826259.042249:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1787826259.042709:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1787826259.042772:0:65912:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826259.042776:0:65912:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826259.042780:0:65912:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1fd:0x0] 00000008:00010000:1.0:1787826259.042801:0:65912:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826259.042804:0:65912:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c32ab0, osc lock ffff907d962f5248, flags 0x400000400001002 00010000:00010000:1.0:1787826259.042825:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f7e3 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826259.042839:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f7e3 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1787826259.042852:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826259.042856:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f7e3 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826259.042865:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f7e3 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1787826259.042875:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826259.042884:0:65912:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:1.0:1787826259.042888:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_release():245 00000020:00010000:1.0:1787826259.042891:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826259.042895:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c32ab0 (ffff907da56082d8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826259.042899:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826259.042902:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c32ab0 (ffff907da5608368/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826259.042913:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/1, layout version 6 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:1.0:1787826259.042963:0:65912:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148) cmd=c00c66f3 arg=aed2a0 00000002:00010000:1.0:1787826259.042984:0:65912:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1fd:0x0]) in obj [0x200000404:0x1fd:0x0], intent: open flags 05004040000002 00010000:00010000:1.0:1787826259.043027:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b6cf lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 65912 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.043035:0:65912:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x81000000001000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b6cf lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x0 pid: 65912 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826259.043046:0:65912:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826259.043057:0:65912:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a12c0 x1874670011397632/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826259.043068:0:65912:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b6cf lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x81000000000000 pid: 65912 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826259.043242:0:65912:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011397632:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826259.043528:0:65912:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb600 going to sleep for 13 seconds 00000100:00100000:1.0:1787826259.051314:0:65912:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011397632:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826259.051340:0:65912:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PW ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b6cf lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc0f806 expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.051354:0:65912:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b6cf lrc: 4/1,0 mode: --/PW res: [0x200000404:0x1fd:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc0f806 expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.051377:0:65912:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b6cf lrc: 4/1,0 mode: PW/PW res: [0x200000404:0x1fd:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0f806 expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.051394:0:65912:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b6cf lrc: 4/1,0 mode: PW/PW res: [0x200000404:0x1fd:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc0f806 expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.051409:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b6cf lrc: 5/1,1 mode: PW/PW res: [0x200000404:0x1fd:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0f806 expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.051418:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b6cf lrc: 5/1,1 mode: PW/PW res: [0x200000404:0x1fd:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0f806 expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.051426:0:65912:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b6cf lrc: 4/0,1 mode: PW/PW res: [0x200000404:0x1fd:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0f806 expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826259.051438:0:65912:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x600002b, status=0 req@ffff907d908a12c0 x1874670011397632/t8589942945(8589942945) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1160 e 0 to 0 dl 1787826277 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826259.051451:0:65912:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a12c0 x1874670011397632/t8589942945(8589942945) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1160 e 0 to 0 dl 1787826277 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826259.051466:0:65912:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b6cf lrc: 3/0,1 mode: PW/PW res: [0x200000404:0x1fd:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc0f806 expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.051480:0:65912:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 2 res 8589935620/509 (0 0) 00000080:00200000:1.0:1787826259.051487:0:65912:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1fd:0x0] suppgids 0 0: rc 0 00000002:00100000:1.0:1787826259.051504:0:65912:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a12c0 x1874670011397632/t8589942945(8589942945) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826277 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1787826259.051512:0:65912:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) for lock 0xe5f736940357b6cf 00000080:00010000:1.0:1787826259.051519:0:65912:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b6cf from it ffff9dd14195bcc0 00010000:00010000:1.0:1787826259.051523:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b6cf lrc: 3/0,1 mode: PW/PW res: [0x200000404:0x1fd:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0f806 expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.051531:0:65912:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b6cf lrc: 2/0,0 mode: PW/PW res: [0x200000404:0x1fd:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0f806 expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1787826259.051595:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 6 restore needed 0 00020000:00200000:1.0:1787826259.051641:0:65912:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826259.051646:0:65912:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826259.051652:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1787826259.051661:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000100:00100000:1.0:1787826259.051715:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011397760:192.168.203.104@tcp:1:lfs.0 00000100:00100000:0.0:1787826259.051879:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011397376:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1787826259.057486:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826259.057592:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025480704 00000100:00100000:2.0:1787826259.057601:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025480704 00000100:00100000:2.0:1787826259.057834:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7126 00000100:00100000:2.0:1787826259.057845:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83525e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:36728:x1874670025480704:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826259.057864:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0f7e3 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826259.057948:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826259.057955:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83525e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:36728:x1874670025480704:12345-192.168.203.104@tcp:104: Request processed in 113us (479us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826259.057963:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7126 00010000:00010000:0.0:1787826259.058009:0:40347:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826259.058017:0:40347:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0f7e3 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826259.058026:0:40347:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907dc2177400 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826259.058032:0:40347:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800428400000000 nid: local remote: 0x7918213f4fc0f7e3 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826259.058052:0:40347:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1787826259.058065:0:40347:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1787826259.058118:0:40347:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 2/0,0 mode: --/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0f7e3 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826259.058127:0:40347:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826259.058164:0:40347:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0f7e3 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826259.058172:0:40347:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000000cf3fb77/0xe5f736940357b6c1 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0f7e3 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826259.058195:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826259.058197:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826259.058199:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826259.058219:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1787826259.058232:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011397888:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826259.063417:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011397888:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826259.063488:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011397760:192.168.203.104@tcp:1:lfs.0 00010000:00010000:1.0:1787826259.063560:0:65912:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1787826259.063593:0:65912:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x1fd:0x0]: losing layout lock 00010000:00010000:1.0:1787826259.063829:0:65912:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1787826259.063844:0:65912:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x1fd:0x0]: losing layout lock 00010000:00010000:1.0:1787826259.063876:0:65912:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 1/0,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826259.063885:0:65912:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 1/0,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826259.063891:0:65912:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1787826259.063895:0:65912:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907dc2177600/0xe5f736940357b6ba lrc: 0/0,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0f7dc expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826259.063913:0:65912:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b689 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0f70a expref: -99 pid: 65896 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1787826259.063928:0:65912:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1787826259.063936:0:65912:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a2d00 x1874670011398144/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 472/440 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826259.063958:0:65912:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011398144:192.168.203.104@tcp:36:lfs.0 00000100:00100000:1.0:1787826259.064067:0:65912:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb600 going to sleep for 13 seconds 00000100:00100000:1.0:1787826259.073488:0:65912:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589942946 gen 1 00000100:00100000:1.0:1787826259.073506:0:65912:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d908a3480 x1874670011396224/t8589942943(8589942943) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826276 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1787826259.073519:0:65912:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d90846d00 x1874670011397248/t8589942944(8589942944) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826277 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1787826259.073529:0:65912:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d908a12c0 x1874670011397632/t8589942945(8589942945) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826277 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1787826259.073540:0:65912:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011398144:192.168.203.104@tcp:36:lfs.0 00010000:00010000:1.0:1787826259.073591:0:65912:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 31 res 8589935620/509 (0 0) 00000080:00010000:1.0:1787826259.073603:0:65912:0:(file.c:7076:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000404:0x1fd:0x0](ffff907d92219148) 00000002:00010000:1.0:1787826259.073628:0:65912:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1fd:0x0]) in obj [0x200000404:0x1fd:0x0], intent: layout flags 0 00010000:00010000:1.0:1787826259.073670:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 65912 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.073679:0:65912:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65912 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.073690:0:65912:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65912 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826259.073709:0:65912:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011398272:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826259.073889:0:65912:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb600 going to sleep for 13 seconds 00000100:00100000:1.0:1787826259.077286:0:65912:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011398272:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826259.077318:0:65912:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.077333:0:65912:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826259.077347:0:65912:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x400 disposition=0x1, status=0 req@ffff907d908a2d00 x1874670011398272/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/896 e 0 to 0 dl 1787826277 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826259.077421:0:65912:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826259.077438:0:65912:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.077451:0:65912:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/509 (0 0) 00000080:00200000:1.0:1787826259.077457:0:65912:0:(namei.c:952:ll_intent_lock()) intent lock 1024 on i1 [0x200000404:0x1fd:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1787826259.077473:0:65912:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) for lock 0xe5f736940357b6d6 00000080:00010000:1.0:1787826259.077481:0:65912:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b6d6 from it ffff9dd14195bc70 00010000:00010000:1.0:1787826259.077485:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.077500:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826259.077522:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826259.077533:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:1.0:1787826259.077540:0:65912:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x1fd:0x0](00000000a294d51b) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826259.077590:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826259.077598:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826259.077634:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826259.077645:0:65912:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826259.077648:0:65912:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826259.077654:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1787826259.077662:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:1.0:1787826259.077679:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1787826259.077691:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826259.077705:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:1.0:1787826259.077713:0:65912:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x1fd:0x0](00000000a294d51b) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826259.077836:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826259.077842:0:65912:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826259.077845:0:65912:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826259.077847:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1787826259.077876:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:1.0:1787826259.077883:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826259.077949:0:65912:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1fd:0x0]: layout version change: 4294967294 -> 7 00010000:00010000:1.0:1787826259.077954:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826259.077964:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200080:1.0:1787826259.077983:0:65912:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148) cmd=66f4 arg=4100000 00000080:00200008:1.0:1787826259.077995:0:65912:0:(file.c:6313:ll_getattr_dentry()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0](ffff907d92219148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826259.078002:0:65912:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148),name=f204a.sanity-flr 00000002:00010000:1.0:1787826259.078014:0:65912:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1fd:0x0]) in obj [0x200000404:0x1fd:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826259.078021:0:65912:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 30 res 8589935620/509 (0 0) 00010000:00010000:1.0:1787826259.078056:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 65912 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.078068:0:65912:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 65912 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.078193:0:65912:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 65912 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826259.078216:0:65912:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011398400:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826259.078356:0:65912:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb600 going to sleep for 13 seconds 00000100:00100000:1.0:1787826259.081926:0:65912:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011398400:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826259.081949:0:65912:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.081961:0:65912:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.081973:0:65912:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.081986:0:65912:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.081998:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.082006:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.082018:0:65912:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826259.082029:0:65912:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d908a2d00 x1874670011398400/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/1136 e 0 to 0 dl 1787826277 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826259.082041:0:65912:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826259.082054:0:65912:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.082067:0:65912:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/509 (0 0) 00000080:00200000:1.0:1787826259.082073:0:65912:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1fd:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826259.082253:0:65912:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1fd:0x0]: layout version change: 7 -> 7 00000080:00010000:1.0:1787826259.082259:0:65912:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) for lock 0xe5f736940357b6dd 00000080:00010000:1.0:1787826259.082264:0:65912:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) 00000080:00010000:1.0:1787826259.082267:0:65912:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b6dd from it ffff9dd14195bcd8 00010000:00010000:1.0:1787826259.082271:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.082281:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826259.082311:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/1, layout version 6 restore needed 0 00020000:00200000:1.0:1787826259.082321:0:65912:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826259.082324:0:65912:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826259.082328:0:65912:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1fd:0x0] 00000008:00010000:1.0:1787826259.082344:0:65912:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826259.082347:0:65912:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c326b0, osc lock ffff907d962f5248, flags 0x400000400001002 00010000:00010000:1.0:1787826259.082453:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.082464:0:65912:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.082475:0:65912:0:(ldlm_request.c:1139:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff907d908a2d00 x1874670011398528/t0(0) o101->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1787826259.082545:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011398528:192.168.203.104@tcp:101:lfs.0 00000020:00200000:1.0:1787826259.086119:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1787826259.086514:0:2348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.086538:0:2348:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.086553:0:2348:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.086566:0:2348:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826259.086597:0:2348:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1787826259.086701:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1787826259.086716:0:2348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.086727:0:2348:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.086735:0:2348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.086745:0:2348:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1787826259.086764:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011398528:192.168.203.104@tcp:101:lfs.0 00000080:00200000:1.0:1787826259.086834:0:65912:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:1.0:1787826259.086840:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_release():245 00000020:00010000:1.0:1787826259.086844:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826259.086849:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c326b0 (ffff907da56082d8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826259.086853:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826259.086856:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c326b0 (ffff907da5608368/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826259.086869:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:1.0:1787826259.122904:0:65912:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148) cmd=400866b1 arg=1 00000080:00200008:1.0:1787826259.122924:0:65912:0:(file.c:6313:ll_getattr_dentry()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0](ffff907d92219148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826259.122933:0:65912:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148),name=f204a.sanity-flr 00000002:00010000:1.0:1787826259.122955:0:65912:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1fd:0x0]) in obj [0x200000404:0x1fd:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826259.122974:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826259.122989:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826259.123003:0:65912:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1fd:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826259.123013:0:65912:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) 00000080:00010000:1.0:1787826259.123016:0:65912:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) for lock 0xe5f736940357b6dd 00000080:00010000:1.0:1787826259.123023:0:65912:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b6dd from it ffff9dd14195bcd8 00010000:00010000:1.0:1787826259.123027:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826259.123036:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826259.123062:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:1.0:1787826259.123074:0:65912:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826259.123088:0:65912:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826259.123092:0:65912:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1fd:0x0] 00000008:00010000:1.0:1787826259.123110:0:65912:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826259.123114:0:65912:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c326b0, osc lock ffff907d962f5248, flags 0x400000400001002 00010000:00010000:1.0:1787826259.123130:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826259.123140:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1787826259.123271:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826259.123276:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826259.123290:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1787826259.123300:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826259.123311:0:65912:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:1.0:1787826259.123315:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_release():245 00000020:00010000:1.0:1787826259.123318:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826259.123322:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c326b0 (ffff907da56082d8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826259.123326:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826259.123328:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c326b0 (ffff907da5608368/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826259.123339:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826259.123356:0:65912:0:(file.c:5224:ll_file_seek()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0], offset: 0, type: SEEK_DATA 00000080:00200000:1.0:1787826259.123425:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:1.0:1787826259.123430:0:65912:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826259.123433:0:65912:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826259.123438:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1787826259.123442:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:1.0:1787826259.123455:0:65912:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000080:00200000:1.0:1787826259.123463:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826259.123469:0:65912:0:(file.c:1848:ll_io_set_mirror()) f204a.sanity-flr: desiginated mirror: 1 00000080:00200000:1.0:1787826259.123471:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826259.123476:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1787826259.123479:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:1.0:1787826259.123482:0:65912:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 18446744073709551615] 00000008:00010000:1.0:1787826259.123488:0:65912:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:1.0:1787826259.123490:0:65912:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c326b0, osc lock ffff907d962f5248, flags 0x0 00010000:00010000:1.0:1787826259.123499:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826259.123510:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826259.123520:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826259.123528:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826259.123536:0:65912:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826259.123546:0:65912:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000100:00100000:0.0:1787826259.123661:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011398656:192.168.203.104@tcp:23:lfs.0 00000100:00100000:0.0:1787826259.127110:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011398656:192.168.203.104@tcp:23:lfs.0 00000020:00010000:1.0:1787826259.127294:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d80024c68 (ffff907da5608368/0) at cl_lock_release():245 00000020:00010000:1.0:1787826259.127308:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d80024c68 (ffff907da5608368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826259.127313:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c326b0 (ffff907da56082d8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1787826259.127324:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826259.127339:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00010000:1.0:1787826259.127356:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d80024c68 (ffff907da5608368/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826259.127359:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c326b0 (ffff907da5608368/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826259.127382:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826259.127391:0:65912:0:(file.c:5262:ll_file_seek()) COMPLETED file f204a.sanity-flr:[0x200000404:0x1fd:0x0], offset: 0, type: SEEK_DATA, rc = 0 00000080:00200008:1.0:1787826259.127405:0:65912:0:(file.c:5224:ll_file_seek()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0], offset: 0, type: SEEK_HOLE 00000080:00200000:1.0:1787826259.127414:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:1.0:1787826259.127423:0:65912:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826259.127426:0:65912:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826259.127431:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1787826259.127437:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:1.0:1787826259.127454:0:65912:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000080:00200000:1.0:1787826259.127460:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826259.127465:0:65912:0:(file.c:1848:ll_io_set_mirror()) f204a.sanity-flr: desiginated mirror: 1 00000080:00200000:1.0:1787826259.127468:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826259.127473:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1787826259.127476:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:1.0:1787826259.127479:0:65912:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 18446744073709551615] 00000008:00010000:1.0:1787826259.127489:0:65912:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:1.0:1787826259.127493:0:65912:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c326b0, osc lock ffff907d962f5248, flags 0x0 00010000:00010000:1.0:1787826259.127513:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826259.127527:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826259.127539:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826259.127548:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826259.127556:0:65912:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1787826259.127580:0:65912:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000100:00100000:0.0:1787826259.127763:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011398784:192.168.203.104@tcp:23:lfs.0 00000100:00100000:0.0:1787826259.131320:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011398784:192.168.203.104@tcp:23:lfs.0 00000020:00010000:1.0:1787826259.131437:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d80024c68 (ffff907da5608368/0) at cl_lock_release():245 00000020:00010000:1.0:1787826259.131442:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d80024c68 (ffff907da5608368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826259.131447:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c326b0 (ffff907da56082d8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1787826259.131457:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826259.131473:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:1.0:1787826259.131490:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d80024c68 (ffff907da5608368/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826259.131493:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c326b0 (ffff907da5608368/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826259.131509:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826259.131517:0:65912:0:(file.c:5262:ll_file_seek()) COMPLETED file f204a.sanity-flr:[0x200000404:0x1fd:0x0], offset: 0, type: SEEK_HOLE, rc = 2097152 00000080:00200080:1.0:1787826259.131532:0:65912:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148) cmd=400866b1 arg=0 00000080:00200080:1.0:1787826259.131541:0:65912:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148) cmd=400866b1 arg=1 00000080:00200008:1.0:1787826259.131548:0:65912:0:(file.c:6313:ll_getattr_dentry()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0](ffff907d92219148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826259.131556:0:65912:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148),name=f204a.sanity-flr 00000002:00010000:1.0:1787826259.131582:0:65912:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1fd:0x0]) in obj [0x200000404:0x1fd:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826259.131598:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826259.131678:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826259.131694:0:65912:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1fd:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826259.131701:0:65912:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) 00000080:00010000:1.0:1787826259.131705:0:65912:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) for lock 0xe5f736940357b6dd 00000080:00010000:1.0:1787826259.131711:0:65912:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b6dd from it ffff9dd14195bcd8 00010000:00010000:1.0:1787826259.131715:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826259.131723:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1787826259.131745:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:1.0:1787826259.131756:0:65912:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826259.131759:0:65912:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826259.131763:0:65912:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1fd:0x0] 00000008:00010000:1.0:1787826259.131776:0:65912:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826259.131780:0:65912:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c326b0, osc lock ffff907d962f5248, flags 0x400000400001002 00010000:00010000:1.0:1787826259.131795:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826259.131804:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1787826259.131814:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826259.131817:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826259.131830:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1787826259.131839:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826259.131848:0:65912:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:1.0:1787826259.131852:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_release():245 00000020:00010000:1.0:1787826259.131854:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826259.131857:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c326b0 (ffff907da56082d8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826259.131860:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826259.131863:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c326b0 (ffff907da5608368/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826259.131869:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826259.131892:0:65912:0:(file.c:2485:do_file_read_iter()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0], ppos: 0, count: 2097152 00000080:00200000:1.0:1787826259.131900:0:65912:0:(file.c:2090:ll_file_io_generic()) f204a.sanity-flr: read ppos: 0, bytes: 2097152 00000080:00200000:1.0:1787826259.131908:0:65912:0:(file.c:1848:ll_io_set_mirror()) f204a.sanity-flr: desiginated mirror: 1 00000020:00200000:1.0:1787826259.131914:0:65912:0:(cl_io.c:218:cl_io_rw_init()) header@000000007d7e8202[0x110, 2, [0x200000404:0x1fd:0x0] hash] 00000020:00200000:1.0:1787826259.131916:0:65912:0:(cl_io.c:218:cl_io_rw_init()) io range: 1 [0, 2097152) 2048 0 00000080:00200000:1.0:1787826259.131919:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00000080:00200000:1.0:1787826259.131927:0:65912:0:(file.c:2170:ll_file_io_generic()) Range lock [0, 511] 00020000:00200000:1.0:1787826259.131934:0:65912:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 2097152) 2097152, 2097152 00020000:00200000:1.0:1787826259.131938:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1787826259.131948:0:65912:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 511] 00000008:00010000:1.0:1787826259.131955:0:65912:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 11 00000008:00010000:1.0:1787826259.131958:0:65912:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c326b0, osc lock ffff907d962f5248, flags 0x40000 00000080:00200000:1.0:1787826259.131963:0:65912:0:(vvp_io.c:874:vvp_io_read_start()) f204a.sanity-flr: read [0, 2097152) 00000080:00200000:1.0:1787826259.131968:0:65912:0:(vvp_io.c:194:vvp_prep_size()) [0x200000404:0x1fd:0x0] updating i_size 2097152 00000080:00200000:1.0:1787826259.131977:0:65912:0:(vvp_io.c:932:vvp_io_read_start()) count:2097152 iocb pos:0 00000080:00200000:1.0:1787826259.131982:0:65912:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148), size=2097152 (max 335544320), offset=0=0x0, pages 512 (max 81920), lockless, parallel 00000020:00200000:1.0:1787826259.132600:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 512 items remaining. 00000080:00200000:1.0:1787826259.132605:0:65912:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 2097152 count 2097152 file_offset 2097152 00000020:00010000:1.0:1787826259.132776:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d80024c68 (ffff907da5608368/0) at cl_lock_release():245 00000020:00010000:1.0:1787826259.132780:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d80024c68 (ffff907da5608368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826259.132783:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c326b0 (ffff907da56082d8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826259.132786:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d80024c68 (ffff907da5608368/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826259.132789:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c326b0 (ffff907da5608368/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826259.132796:0:65912:0:(vvp_io.c:533:vvp_io_advance()) advancing 2097152 bytes 00000020:00200000:1.0:1787826259.132801:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1787826259.132823:0:49858:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907d908aab40 00000008:00100000:1.0:1787826259.132937:0:49858:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d908a0780 - object 0x240000400:803 offset 0<>2097152 00000100:00100000:0.0:1787826259.133030:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1787826259.133049:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011398912:192.168.203.104@tcp:3:lfs.0 00000008:00010000:0.0:1787826259.160022:0:2348:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907d908aab40 00000020:00200000:0.0:1787826259.160034:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 511 items remaining. 00000020:00200000:0.0:1787826259.160038:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 510 items remaining. 00000020:00200000:0.0:1787826259.160040:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 509 items remaining. 00000020:00200000:0.0:1787826259.160042:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 508 items remaining. 00000020:00200000:0.0:1787826259.160044:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 507 items remaining. 00000020:00200000:0.0:1787826259.160046:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 506 items remaining. 00000020:00200000:0.0:1787826259.160048:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 505 items remaining. 00000020:00200000:0.0:1787826259.160050:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 504 items remaining. 00000020:00200000:0.0:1787826259.160052:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 503 items remaining. 00000020:00200000:0.0:1787826259.160053:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 502 items remaining. 00000020:00200000:0.0:1787826259.160055:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 501 items remaining. 00000020:00200000:0.0:1787826259.160057:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 500 items remaining. 00000020:00200000:0.0:1787826259.160058:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 499 items remaining. 00000020:00200000:0.0:1787826259.160060:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 498 items remaining. 00000020:00200000:0.0:1787826259.160062:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 497 items remaining. 00000020:00200000:0.0:1787826259.160064:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 496 items remaining. 00000020:00200000:0.0:1787826259.160065:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 495 items remaining. 00000020:00200000:0.0:1787826259.160067:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 494 items remaining. 00000020:00200000:0.0:1787826259.160069:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 493 items remaining. 00000020:00200000:0.0:1787826259.160071:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 492 items remaining. 00000020:00200000:0.0:1787826259.160073:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 491 items remaining. 00000020:00200000:0.0:1787826259.160075:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 490 items remaining. 00000020:00200000:0.0:1787826259.160143:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 489 items remaining. 00000020:00200000:0.0:1787826259.160202:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 488 items remaining. 00000020:00200000:0.0:1787826259.160206:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 487 items remaining. 00000020:00200000:0.0:1787826259.160207:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 486 items remaining. 00000020:00200000:0.0:1787826259.160209:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 485 items remaining. 00000020:00200000:0.0:1787826259.160211:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 484 items remaining. 00000020:00200000:0.0:1787826259.160213:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 483 items remaining. 00000020:00200000:0.0:1787826259.160215:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 482 items remaining. 00000020:00200000:0.0:1787826259.160216:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 481 items remaining. 00000020:00200000:0.0:1787826259.160218:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 480 items remaining. 00000020:00200000:0.0:1787826259.160219:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 479 items remaining. 00000020:00200000:0.0:1787826259.160221:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 478 items remaining. 00000020:00200000:0.0:1787826259.160222:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 477 items remaining. 00000020:00200000:0.0:1787826259.160224:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 476 items remaining. 00000020:00200000:0.0:1787826259.160225:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 475 items remaining. 00000020:00200000:0.0:1787826259.160227:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 474 items remaining. 00000020:00200000:0.0:1787826259.160229:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 473 items remaining. 00000020:00200000:0.0:1787826259.160230:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 472 items remaining. 00000020:00200000:0.0:1787826259.160232:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 471 items remaining. 00000020:00200000:0.0:1787826259.160234:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 470 items remaining. 00000020:00200000:0.0:1787826259.160235:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 469 items remaining. 00000020:00200000:0.0:1787826259.160237:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 468 items remaining. 00000020:00200000:0.0:1787826259.160239:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 467 items remaining. 00000020:00200000:0.0:1787826259.160240:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 466 items remaining. 00000020:00200000:0.0:1787826259.160242:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 465 items remaining. 00000020:00200000:0.0:1787826259.160243:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 464 items remaining. 00000020:00200000:0.0:1787826259.160245:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 463 items remaining. 00000020:00200000:0.0:1787826259.160246:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 462 items remaining. 00000020:00200000:0.0:1787826259.160248:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 461 items remaining. 00000020:00200000:0.0:1787826259.160250:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 460 items remaining. 00000020:00200000:0.0:1787826259.160251:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 459 items remaining. 00000020:00200000:0.0:1787826259.160252:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 458 items remaining. 00000020:00200000:0.0:1787826259.160254:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 457 items remaining. 00000020:00200000:0.0:1787826259.160256:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 456 items remaining. 00000020:00200000:0.0:1787826259.160257:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 455 items remaining. 00000020:00200000:0.0:1787826259.160259:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 454 items remaining. 00000020:00200000:0.0:1787826259.160260:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 453 items remaining. 00000020:00200000:0.0:1787826259.160263:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 452 items remaining. 00000020:00200000:0.0:1787826259.160264:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 451 items remaining. 00000020:00200000:0.0:1787826259.160266:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 450 items remaining. 00000020:00200000:0.0:1787826259.160268:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 449 items remaining. 00000020:00200000:0.0:1787826259.160269:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 448 items remaining. 00000020:00200000:0.0:1787826259.160271:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 447 items remaining. 00000020:00200000:0.0:1787826259.160272:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 446 items remaining. 00000020:00200000:0.0:1787826259.160274:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 445 items remaining. 00000020:00200000:0.0:1787826259.160276:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 444 items remaining. 00000020:00200000:0.0:1787826259.160277:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 443 items remaining. 00000020:00200000:0.0:1787826259.160279:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 442 items remaining. 00000020:00200000:0.0:1787826259.160280:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 441 items remaining. 00000020:00200000:0.0:1787826259.160282:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 440 items remaining. 00000020:00200000:0.0:1787826259.160284:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 439 items remaining. 00000020:00200000:0.0:1787826259.160286:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 438 items remaining. 00000020:00200000:0.0:1787826259.160288:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 437 items remaining. 00000020:00200000:0.0:1787826259.160290:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 436 items remaining. 00000020:00200000:0.0:1787826259.160292:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 435 items remaining. 00000020:00200000:0.0:1787826259.160294:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 434 items remaining. 00000020:00200000:0.0:1787826259.160296:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 433 items remaining. 00000020:00200000:0.0:1787826259.160298:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 432 items remaining. 00000020:00200000:0.0:1787826259.160299:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 431 items remaining. 00000020:00200000:0.0:1787826259.160301:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 430 items remaining. 00000020:00200000:0.0:1787826259.160303:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 429 items remaining. 00000020:00200000:0.0:1787826259.160305:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 428 items remaining. 00000020:00200000:0.0:1787826259.160307:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 427 items remaining. 00000020:00200000:0.0:1787826259.160309:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 426 items remaining. 00000020:00200000:0.0:1787826259.160310:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 425 items remaining. 00000020:00200000:0.0:1787826259.160312:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 424 items remaining. 00000020:00200000:0.0:1787826259.160314:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 423 items remaining. 00000020:00200000:0.0:1787826259.160315:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 422 items remaining. 00000020:00200000:0.0:1787826259.160317:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 421 items remaining. 00000020:00200000:0.0:1787826259.160319:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 420 items remaining. 00000020:00200000:0.0:1787826259.160321:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 419 items remaining. 00000020:00200000:0.0:1787826259.160323:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 418 items remaining. 00000020:00200000:0.0:1787826259.160325:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 417 items remaining. 00000020:00200000:0.0:1787826259.160326:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 416 items remaining. 00000020:00200000:0.0:1787826259.160328:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 415 items remaining. 00000020:00200000:0.0:1787826259.160330:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 414 items remaining. 00000020:00200000:0.0:1787826259.160331:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 413 items remaining. 00000020:00200000:0.0:1787826259.160333:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 412 items remaining. 00000020:00200000:0.0:1787826259.160335:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 411 items remaining. 00000020:00200000:0.0:1787826259.160336:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 410 items remaining. 00000020:00200000:0.0:1787826259.160338:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 409 items remaining. 00000020:00200000:0.0:1787826259.160340:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 408 items remaining. 00000020:00200000:0.0:1787826259.160342:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 407 items remaining. 00000020:00200000:0.0:1787826259.160344:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 406 items remaining. 00000020:00200000:0.0:1787826259.160346:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 405 items remaining. 00000020:00200000:0.0:1787826259.160348:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 404 items remaining. 00000020:00200000:0.0:1787826259.160349:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 403 items remaining. 00000020:00200000:0.0:1787826259.160351:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 402 items remaining. 00000020:00200000:0.0:1787826259.160353:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 401 items remaining. 00000020:00200000:0.0:1787826259.160355:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 400 items remaining. 00000020:00200000:0.0:1787826259.160356:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 399 items remaining. 00000020:00200000:0.0:1787826259.160579:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 398 items remaining. 00000020:00200000:0.0:1787826259.160581:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 397 items remaining. 00000020:00200000:0.0:1787826259.160583:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 396 items remaining. 00000020:00200000:0.0:1787826259.160584:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 395 items remaining. 00000020:00200000:0.0:1787826259.160586:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 394 items remaining. 00000020:00200000:0.0:1787826259.160588:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 393 items remaining. 00000020:00200000:0.0:1787826259.160590:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 392 items remaining. 00000020:00200000:0.0:1787826259.160592:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 391 items remaining. 00000020:00200000:0.0:1787826259.160594:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 390 items remaining. 00000020:00200000:0.0:1787826259.160595:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 389 items remaining. 00000020:00200000:0.0:1787826259.160597:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 388 items remaining. 00000020:00200000:0.0:1787826259.160599:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 387 items remaining. 00000020:00200000:0.0:1787826259.160600:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 386 items remaining. 00000020:00200000:0.0:1787826259.160602:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 385 items remaining. 00000020:00200000:0.0:1787826259.160604:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 384 items remaining. 00000020:00200000:0.0:1787826259.160605:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 383 items remaining. 00000020:00200000:0.0:1787826259.160677:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 382 items remaining. 00000020:00200000:0.0:1787826259.160679:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 381 items remaining. 00000020:00200000:0.0:1787826259.160680:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 380 items remaining. 00000020:00200000:0.0:1787826259.160682:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 379 items remaining. 00000020:00200000:0.0:1787826259.160684:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 378 items remaining. 00000020:00200000:0.0:1787826259.160686:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 377 items remaining. 00000020:00200000:0.0:1787826259.160687:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 376 items remaining. 00000020:00200000:0.0:1787826259.160689:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 375 items remaining. 00000020:00200000:0.0:1787826259.160690:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 374 items remaining. 00000020:00200000:0.0:1787826259.160692:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 373 items remaining. 00000020:00200000:0.0:1787826259.160694:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 372 items remaining. 00000020:00200000:0.0:1787826259.160695:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 371 items remaining. 00000020:00200000:0.0:1787826259.160697:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 370 items remaining. 00000020:00200000:0.0:1787826259.160699:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 369 items remaining. 00000020:00200000:0.0:1787826259.160700:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 368 items remaining. 00000020:00200000:0.0:1787826259.160702:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 367 items remaining. 00000020:00200000:0.0:1787826259.160703:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 366 items remaining. 00000020:00200000:0.0:1787826259.160705:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 365 items remaining. 00000020:00200000:0.0:1787826259.160707:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 364 items remaining. 00000020:00200000:0.0:1787826259.160708:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 363 items remaining. 00000020:00200000:0.0:1787826259.160710:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 362 items remaining. 00000020:00200000:0.0:1787826259.160712:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 361 items remaining. 00000020:00200000:0.0:1787826259.160713:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 360 items remaining. 00000020:00200000:0.0:1787826259.160715:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 359 items remaining. 00000020:00200000:0.0:1787826259.160717:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 358 items remaining. 00000020:00200000:0.0:1787826259.160719:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 357 items remaining. 00000020:00200000:0.0:1787826259.160721:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 356 items remaining. 00000020:00200000:0.0:1787826259.160722:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 355 items remaining. 00000020:00200000:0.0:1787826259.160773:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 354 items remaining. 00000020:00200000:0.0:1787826259.160775:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 353 items remaining. 00000020:00200000:0.0:1787826259.160777:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 352 items remaining. 00000020:00200000:0.0:1787826259.160778:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 351 items remaining. 00000020:00200000:0.0:1787826259.160780:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 350 items remaining. 00000020:00200000:0.0:1787826259.160782:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 349 items remaining. 00000020:00200000:0.0:1787826259.160784:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 348 items remaining. 00000020:00200000:0.0:1787826259.160786:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 347 items remaining. 00000020:00200000:0.0:1787826259.160787:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 346 items remaining. 00000020:00200000:0.0:1787826259.160789:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 345 items remaining. 00000020:00200000:0.0:1787826259.160790:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 344 items remaining. 00000020:00200000:0.0:1787826259.160792:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 343 items remaining. 00000020:00200000:0.0:1787826259.160794:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 342 items remaining. 00000020:00200000:0.0:1787826259.160795:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 341 items remaining. 00000020:00200000:0.0:1787826259.160797:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 340 items remaining. 00000020:00200000:0.0:1787826259.160799:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 339 items remaining. 00000020:00200000:0.0:1787826259.160800:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 338 items remaining. 00000020:00200000:0.0:1787826259.160802:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 337 items remaining. 00000020:00200000:0.0:1787826259.160803:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 336 items remaining. 00000020:00200000:0.0:1787826259.160805:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 335 items remaining. 00000020:00200000:0.0:1787826259.160806:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 334 items remaining. 00000020:00200000:0.0:1787826259.160808:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 333 items remaining. 00000020:00200000:0.0:1787826259.160810:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 332 items remaining. 00000020:00200000:0.0:1787826259.160811:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 331 items remaining. 00000020:00200000:0.0:1787826259.160812:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 330 items remaining. 00000020:00200000:0.0:1787826259.160814:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 329 items remaining. 00000020:00200000:0.0:1787826259.160815:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 328 items remaining. 00000020:00200000:0.0:1787826259.160817:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 327 items remaining. 00000020:00200000:0.0:1787826259.160819:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 326 items remaining. 00000020:00200000:0.0:1787826259.160820:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 325 items remaining. 00000020:00200000:0.0:1787826259.160822:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 324 items remaining. 00000020:00200000:0.0:1787826259.160824:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 323 items remaining. 00000020:00200000:0.0:1787826259.160826:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 322 items remaining. 00000020:00200000:0.0:1787826259.160827:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 321 items remaining. 00000020:00200000:0.0:1787826259.160829:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 320 items remaining. 00000020:00200000:0.0:1787826259.160831:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 319 items remaining. 00000020:00200000:0.0:1787826259.160832:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 318 items remaining. 00000020:00200000:0.0:1787826259.160834:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 317 items remaining. 00000020:00200000:0.0:1787826259.160836:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 316 items remaining. 00000020:00200000:0.0:1787826259.160837:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 315 items remaining. 00000020:00200000:0.0:1787826259.160839:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 314 items remaining. 00000020:00200000:0.0:1787826259.160840:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 313 items remaining. 00000020:00200000:0.0:1787826259.160842:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 312 items remaining. 00000020:00200000:0.0:1787826259.160843:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 311 items remaining. 00000020:00200000:0.0:1787826259.160845:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 310 items remaining. 00000020:00200000:0.0:1787826259.160846:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 309 items remaining. 00000020:00200000:0.0:1787826259.160848:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 308 items remaining. 00000020:00200000:0.0:1787826259.160849:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 307 items remaining. 00000020:00200000:0.0:1787826259.160851:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 306 items remaining. 00000020:00200000:0.0:1787826259.160852:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 305 items remaining. 00000020:00200000:0.0:1787826259.160854:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 304 items remaining. 00000020:00200000:0.0:1787826259.160856:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 303 items remaining. 00000020:00200000:0.0:1787826259.160858:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 302 items remaining. 00000020:00200000:0.0:1787826259.160860:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 301 items remaining. 00000020:00200000:0.0:1787826259.160861:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 300 items remaining. 00000020:00200000:0.0:1787826259.160863:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 299 items remaining. 00000020:00200000:0.0:1787826259.160865:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 298 items remaining. 00000020:00200000:0.0:1787826259.160866:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 297 items remaining. 00000020:00200000:0.0:1787826259.160868:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 296 items remaining. 00000020:00200000:0.0:1787826259.160870:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 295 items remaining. 00000020:00200000:0.0:1787826259.160871:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 294 items remaining. 00000020:00200000:0.0:1787826259.160873:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 293 items remaining. 00000020:00200000:0.0:1787826259.160876:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 292 items remaining. 00000020:00200000:0.0:1787826259.160877:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 291 items remaining. 00000020:00200000:0.0:1787826259.160879:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 290 items remaining. 00000020:00200000:0.0:1787826259.160881:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 289 items remaining. 00000020:00200000:0.0:1787826259.160883:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 288 items remaining. 00000020:00200000:0.0:1787826259.160884:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 287 items remaining. 00000020:00200000:0.0:1787826259.160886:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 286 items remaining. 00000020:00200000:0.0:1787826259.160888:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 285 items remaining. 00000020:00200000:0.0:1787826259.160890:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 284 items remaining. 00000020:00200000:0.0:1787826259.160891:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 283 items remaining. 00000020:00200000:0.0:1787826259.160893:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 282 items remaining. 00000020:00200000:0.0:1787826259.160895:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 281 items remaining. 00000020:00200000:0.0:1787826259.160896:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 280 items remaining. 00000020:00200000:0.0:1787826259.160899:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 279 items remaining. 00000020:00200000:0.0:1787826259.160901:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 278 items remaining. 00000020:00200000:0.0:1787826259.160902:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 277 items remaining. 00000020:00200000:0.0:1787826259.160904:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 276 items remaining. 00000020:00200000:0.0:1787826259.160906:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 275 items remaining. 00000020:00200000:0.0:1787826259.160907:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 274 items remaining. 00000020:00200000:0.0:1787826259.160909:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 273 items remaining. 00000020:00200000:0.0:1787826259.160911:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 272 items remaining. 00000020:00200000:0.0:1787826259.160912:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 271 items remaining. 00000020:00200000:0.0:1787826259.160914:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 270 items remaining. 00000020:00200000:0.0:1787826259.160916:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 269 items remaining. 00000020:00200000:0.0:1787826259.160917:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 268 items remaining. 00000020:00200000:0.0:1787826259.160919:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 267 items remaining. 00000020:00200000:0.0:1787826259.160921:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 266 items remaining. 00000020:00200000:0.0:1787826259.160923:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 265 items remaining. 00000020:00200000:0.0:1787826259.160924:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 264 items remaining. 00000020:00200000:0.0:1787826259.160926:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 263 items remaining. 00000020:00200000:0.0:1787826259.160928:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 262 items remaining. 00000020:00200000:0.0:1787826259.160929:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 261 items remaining. 00000020:00200000:0.0:1787826259.160931:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 260 items remaining. 00000020:00200000:0.0:1787826259.160933:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 259 items remaining. 00000020:00200000:0.0:1787826259.160934:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 258 items remaining. 00000020:00200000:0.0:1787826259.160936:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 257 items remaining. 00000020:00200000:0.0:1787826259.160937:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000020:00200000:0.0:1787826259.160939:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:0.0:1787826259.160940:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:0.0:1787826259.160942:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:0.0:1787826259.160943:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:0.0:1787826259.160945:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:0.0:1787826259.160946:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:0.0:1787826259.160948:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:0.0:1787826259.160950:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:0.0:1787826259.160952:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:0.0:1787826259.160953:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:0.0:1787826259.160954:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:0.0:1787826259.160956:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:0.0:1787826259.160957:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:0.0:1787826259.160959:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:0.0:1787826259.160960:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:0.0:1787826259.160962:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:0.0:1787826259.160963:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:0.0:1787826259.160965:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:0.0:1787826259.160966:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:0.0:1787826259.160968:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:0.0:1787826259.160969:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:0.0:1787826259.160971:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:0.0:1787826259.160972:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:0.0:1787826259.160973:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:0.0:1787826259.160975:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:0.0:1787826259.160976:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:0.0:1787826259.160977:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:0.0:1787826259.160979:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:0.0:1787826259.160980:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:0.0:1787826259.160982:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:0.0:1787826259.160984:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:0.0:1787826259.160985:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:0.0:1787826259.160987:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:0.0:1787826259.160989:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:0.0:1787826259.160990:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:0.0:1787826259.160992:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:0.0:1787826259.160993:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:0.0:1787826259.160995:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:0.0:1787826259.160996:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:0.0:1787826259.160998:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:0.0:1787826259.161000:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:0.0:1787826259.161002:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:0.0:1787826259.161003:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:0.0:1787826259.161005:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:0.0:1787826259.161006:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:0.0:1787826259.161008:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:0.0:1787826259.161009:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:0.0:1787826259.161011:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:0.0:1787826259.161012:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:0.0:1787826259.161014:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:0.0:1787826259.161016:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:0.0:1787826259.161017:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:0.0:1787826259.161018:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:0.0:1787826259.161020:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:0.0:1787826259.161021:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:0.0:1787826259.161023:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:0.0:1787826259.161024:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:0.0:1787826259.161025:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:0.0:1787826259.161027:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:0.0:1787826259.161029:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:0.0:1787826259.161030:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:0.0:1787826259.161031:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:0.0:1787826259.161033:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:0.0:1787826259.161035:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:0.0:1787826259.161036:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:0.0:1787826259.161038:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:0.0:1787826259.161039:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:0.0:1787826259.161041:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:0.0:1787826259.161042:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:0.0:1787826259.161044:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:0.0:1787826259.161046:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:0.0:1787826259.161048:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:0.0:1787826259.161050:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:0.0:1787826259.161051:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:0.0:1787826259.161053:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:0.0:1787826259.161054:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:0.0:1787826259.161056:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:0.0:1787826259.161057:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:0.0:1787826259.161059:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:0.0:1787826259.161061:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:0.0:1787826259.161062:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:0.0:1787826259.161065:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:0.0:1787826259.161067:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:0.0:1787826259.161068:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:0.0:1787826259.161070:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:0.0:1787826259.161072:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:0.0:1787826259.161074:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:0.0:1787826259.161075:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:0.0:1787826259.161139:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:0.0:1787826259.161200:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:0.0:1787826259.161202:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:0.0:1787826259.161204:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:0.0:1787826259.161206:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:0.0:1787826259.161208:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:0.0:1787826259.161209:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:0.0:1787826259.161211:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:0.0:1787826259.161212:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:0.0:1787826259.161214:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:0.0:1787826259.161215:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:0.0:1787826259.161217:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:0.0:1787826259.161219:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:0.0:1787826259.161220:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:0.0:1787826259.161222:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:0.0:1787826259.161223:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:0.0:1787826259.161225:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:0.0:1787826259.161227:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:0.0:1787826259.161228:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:0.0:1787826259.161230:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:0.0:1787826259.161232:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:0.0:1787826259.161234:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:0.0:1787826259.161236:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:0.0:1787826259.161238:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:0.0:1787826259.161240:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:0.0:1787826259.161241:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:0.0:1787826259.161243:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:0.0:1787826259.161245:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:0.0:1787826259.161246:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:0.0:1787826259.161248:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:0.0:1787826259.161249:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:0.0:1787826259.161251:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:0.0:1787826259.161253:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:0.0:1787826259.161254:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:0.0:1787826259.161256:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:0.0:1787826259.161258:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:0.0:1787826259.161260:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:0.0:1787826259.161261:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:0.0:1787826259.161264:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:0.0:1787826259.161265:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:0.0:1787826259.161267:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:0.0:1787826259.161268:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:0.0:1787826259.161270:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:0.0:1787826259.161272:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:0.0:1787826259.161274:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:0.0:1787826259.161276:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:0.0:1787826259.161277:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:0.0:1787826259.161279:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:0.0:1787826259.161281:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:0.0:1787826259.161282:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:0.0:1787826259.161284:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:0.0:1787826259.161285:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:0.0:1787826259.161287:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:0.0:1787826259.161289:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:0.0:1787826259.161290:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:0.0:1787826259.161292:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:0.0:1787826259.161294:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:0.0:1787826259.161296:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:0.0:1787826259.161298:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:0.0:1787826259.161300:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:0.0:1787826259.161302:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:0.0:1787826259.161304:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:0.0:1787826259.161306:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:0.0:1787826259.161308:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:0.0:1787826259.161310:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:0.0:1787826259.161311:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:0.0:1787826259.161313:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:0.0:1787826259.161315:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:0.0:1787826259.161318:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:0.0:1787826259.161320:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:0.0:1787826259.161322:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:0.0:1787826259.161323:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:0.0:1787826259.161325:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:0.0:1787826259.161326:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:0.0:1787826259.161328:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:0.0:1787826259.161330:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:0.0:1787826259.161332:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:0.0:1787826259.161334:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:0.0:1787826259.161336:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:0.0:1787826259.161338:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:0.0:1787826259.161340:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:0.0:1787826259.161341:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:0.0:1787826259.161343:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:0.0:1787826259.161345:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:0.0:1787826259.161347:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:0.0:1787826259.161349:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:0.0:1787826259.161350:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:0.0:1787826259.161352:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:0.0:1787826259.161354:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:0.0:1787826259.161356:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:0.0:1787826259.161418:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:0.0:1787826259.161421:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:0.0:1787826259.161422:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:0.0:1787826259.161424:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:0.0:1787826259.161426:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:0.0:1787826259.161428:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:0.0:1787826259.161430:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:0.0:1787826259.161432:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:0.0:1787826259.161434:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:0.0:1787826259.161436:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:0.0:1787826259.161438:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:0.0:1787826259.161440:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:0.0:1787826259.161442:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:0.0:1787826259.161443:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:0.0:1787826259.161445:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:0.0:1787826259.161447:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:0.0:1787826259.161449:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:0.0:1787826259.161450:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:0.0:1787826259.161452:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:0.0:1787826259.161454:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:0.0:1787826259.161456:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:0.0:1787826259.161458:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:0.0:1787826259.161460:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:0.0:1787826259.161462:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:0.0:1787826259.161464:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:0.0:1787826259.161465:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:0.0:1787826259.161467:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:0.0:1787826259.161469:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:0.0:1787826259.161470:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:0.0:1787826259.161472:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:0.0:1787826259.161474:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:0.0:1787826259.161475:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:0.0:1787826259.161477:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:0.0:1787826259.161478:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:0.0:1787826259.161480:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:0.0:1787826259.161482:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:0.0:1787826259.161484:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:0.0:1787826259.161486:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:0.0:1787826259.161488:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:0.0:1787826259.161490:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:0.0:1787826259.161492:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:0.0:1787826259.161494:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:0.0:1787826259.161496:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:0.0:1787826259.161498:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:0.0:1787826259.161500:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:0.0:1787826259.161501:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:0.0:1787826259.161503:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:0.0:1787826259.161505:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:0.0:1787826259.161507:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:0.0:1787826259.161508:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:0.0:1787826259.161510:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:0.0:1787826259.161512:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:0.0:1787826259.161513:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:0.0:1787826259.161515:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:0.0:1787826259.161517:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:0.0:1787826259.161519:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:0.0:1787826259.161520:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:0.0:1787826259.161522:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:0.0:1787826259.161524:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:0.0:1787826259.161526:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:0.0:1787826259.161527:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:0.0:1787826259.161529:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:0.0:1787826259.161531:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:0.0:1787826259.161533:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:0.0:1787826259.161534:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:0.0:1787826259.161536:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:0.0:1787826259.161538:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:0.0:1787826259.161540:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:0.0:1787826259.161541:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:0.0:1787826259.161543:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:0.0:1787826259.161545:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:0.0:1787826259.161546:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:0.0:1787826259.161548:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:0.0:1787826259.161551:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:0.0:1787826259.161552:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:0.0:1787826259.161554:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:0.0:1787826259.161556:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:0.0:1787826259.161557:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1787826259.162060:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1787826259.162137:0:2348:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907d908aab40 00000100:00100000:0.0:1787826259.162174:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011398912:192.168.203.104@tcp:3:lfs.0 00000080:00200000:1.0:1787826259.162418:0:65912:0:(file.c:2207:ll_file_io_generic()) Range unlock [0, 511] 00000080:00200000:1.0:1787826259.162430:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826259.162435:0:65912:0:(file.c:2240:ll_file_io_generic()) f204a.sanity-flr: 1 io complete with rc: 0, result: 2097152, restart: 0 00000020:00200000:1.0:1787826259.162438:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826259.162446:0:65912:0:(file.c:2326:ll_file_io_generic()) iot: 1, result: 2097152 00000080:00200080:1.0:1787826259.162462:0:65912:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148) cmd=400866b1 arg=0 00000080:00200080:1.0:1787826259.162472:0:65912:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148) cmd=400866b1 arg=2 00000080:00200008:1.0:1787826259.162479:0:65912:0:(file.c:6313:ll_getattr_dentry()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0](ffff907d92219148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826259.162486:0:65912:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148),name=f204a.sanity-flr 00000002:00010000:1.0:1787826259.162503:0:65912:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1fd:0x0]) in obj [0x200000404:0x1fd:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826259.162521:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826259.162535:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1787826259.162543:0:65912:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1fd:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826259.162550:0:65912:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) 00000080:00010000:1.0:1787826259.162554:0:65912:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) for lock 0xe5f736940357b6dd 00000080:00010000:1.0:1787826259.162562:0:65912:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b6dd from it ffff9dd14195bcd8 00010000:00010000:1.0:1787826259.162576:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826259.162586:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1787826259.162640:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:1.0:1787826259.162651:0:65912:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826259.162654:0:65912:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826259.162658:0:65912:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1fd:0x0] 00000008:00010000:1.0:1787826259.162675:0:65912:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826259.162679:0:65912:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c326b0, osc lock ffff907d962f5248, flags 0x400000400001002 00010000:00010000:1.0:1787826259.162695:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826259.162706:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1787826259.162717:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826259.162721:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826259.162827:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00200000:1.0:1787826259.162839:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826259.162849:0:65912:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:1.0:1787826259.162854:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_release():245 00000020:00010000:1.0:1787826259.162858:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826259.162861:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c326b0 (ffff907da56082d8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826259.162865:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826259.162868:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c326b0 (ffff907da5608368/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826259.162876:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826259.162897:0:65912:0:(file.c:2652:do_file_write_iter()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0], ppos: 0, count: 1048576 00000080:00200000:1.0:1787826259.162903:0:65912:0:(file.c:2090:ll_file_io_generic()) f204a.sanity-flr: write ppos: 0, bytes: 1048576 00000080:00200000:1.0:1787826259.162910:0:65912:0:(file.c:1848:ll_io_set_mirror()) f204a.sanity-flr: desiginated mirror: 2 00000020:00200000:1.0:1787826259.162917:0:65912:0:(cl_io.c:218:cl_io_rw_init()) header@000000007d7e8202[0x110, 2, [0x200000404:0x1fd:0x0] hash] 00000020:00200000:1.0:1787826259.162919:0:65912:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [0, 1048576) 2048 0 00000080:00200000:1.0:1787826259.162922:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00000080:00200000:1.0:1787826259.162931:0:65912:0:(file.c:2170:ll_file_io_generic()) Range lock [0, 255] 00020000:00200000:1.0:1787826259.162938:0:65912:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:1.0:1787826259.162942:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x11 00000080:00200000:1.0:1787826259.162954:0:65912:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 255] 00000008:00010000:1.0:1787826259.162961:0:65912:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 11 00000008:00010000:1.0:1787826259.162964:0:65912:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c326b0, osc lock ffff907d962f5248, flags 0x40000 00000080:00200000:1.0:1787826259.162970:0:65912:0:(vvp_io.c:1339:vvp_io_write_start()) f204a.sanity-flr: write [0, 1048576) append 0 append lockpos 0 00000080:00200000:1.0:1787826259.162995:0:65912:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148), size=1048576 (max 335544320), offset=0=0x0, pages 256 (max 81920), lockless, parallel 00000020:00200000:1.0:1787826259.163473:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:1.0:1787826259.163477:0:65912:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 1048576 00000080:00200000:1.0:1787826259.163486:0:65912:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:1.0:1787826259.163490:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d80024c68 (ffff907da5608368/0) at cl_lock_release():245 00000020:00010000:1.0:1787826259.163493:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d80024c68 (ffff907da5608368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826259.163496:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c326b0 (ffff907da56082d8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826259.163499:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d80024c68 (ffff907da5608368/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826259.163501:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c326b0 (ffff907da5608368/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826259.163507:0:65912:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:1.0:1787826259.163512:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1787826259.163532:0:49858:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907d908aad80 00000008:00100000:1.0:1787826259.164441:0:49858:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d908a0780 - object 0x280000400:775 offset 0<>1048576 00000100:00100000:0.0:1787826259.164519:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011399040:192.168.203.104@tcp:4:lfs.0 00000100:00100000:0.0:1787826259.190732:0:2348:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0001-osc-ffff907d986d0000: committing for last_committed 17179873820 gen 1 00000008:00010000:0.0:1787826259.190813:0:2348:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907d908aad80 00000020:00200000:0.0:1787826259.190820:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:0.0:1787826259.190823:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:0.0:1787826259.190824:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:0.0:1787826259.190826:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:0.0:1787826259.190827:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:0.0:1787826259.190829:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:0.0:1787826259.190831:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:0.0:1787826259.190833:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:0.0:1787826259.190835:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:0.0:1787826259.190836:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:0.0:1787826259.190838:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:0.0:1787826259.190840:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:0.0:1787826259.190842:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:0.0:1787826259.190844:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:0.0:1787826259.190845:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:0.0:1787826259.190847:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:0.0:1787826259.190848:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:0.0:1787826259.190850:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:0.0:1787826259.190853:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:0.0:1787826259.190855:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:0.0:1787826259.190856:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:0.0:1787826259.190858:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:0.0:1787826259.190860:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:0.0:1787826259.190861:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:0.0:1787826259.190863:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:0.0:1787826259.190865:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:0.0:1787826259.190866:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:0.0:1787826259.190867:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:0.0:1787826259.190869:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:0.0:1787826259.190871:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:0.0:1787826259.190872:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:0.0:1787826259.190874:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:0.0:1787826259.190876:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:0.0:1787826259.190878:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:0.0:1787826259.190880:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:0.0:1787826259.190881:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:0.0:1787826259.190883:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:0.0:1787826259.190885:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:0.0:1787826259.190887:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:0.0:1787826259.190889:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:0.0:1787826259.190891:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:0.0:1787826259.190892:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:0.0:1787826259.190894:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:0.0:1787826259.190896:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:0.0:1787826259.190897:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:0.0:1787826259.190899:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:0.0:1787826259.190901:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:0.0:1787826259.190903:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:0.0:1787826259.190905:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:0.0:1787826259.190907:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:0.0:1787826259.190908:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:0.0:1787826259.190910:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:0.0:1787826259.190912:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:0.0:1787826259.190914:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:0.0:1787826259.190916:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:0.0:1787826259.190918:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:0.0:1787826259.190919:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:0.0:1787826259.190921:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:0.0:1787826259.190923:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:0.0:1787826259.190924:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:0.0:1787826259.190926:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:0.0:1787826259.190928:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:0.0:1787826259.190929:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:0.0:1787826259.190932:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:0.0:1787826259.190933:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:0.0:1787826259.190935:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:0.0:1787826259.190937:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:0.0:1787826259.190939:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:0.0:1787826259.190940:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:0.0:1787826259.190942:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:0.0:1787826259.190944:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:0.0:1787826259.190946:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:0.0:1787826259.190947:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:0.0:1787826259.190949:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:0.0:1787826259.190950:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:0.0:1787826259.190952:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:0.0:1787826259.190954:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:0.0:1787826259.190956:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:0.0:1787826259.190957:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:0.0:1787826259.190959:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:0.0:1787826259.190960:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:0.0:1787826259.190962:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:0.0:1787826259.190964:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:0.0:1787826259.190966:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:0.0:1787826259.190968:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:0.0:1787826259.190970:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:0.0:1787826259.190971:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:0.0:1787826259.190973:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:0.0:1787826259.190975:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:0.0:1787826259.190976:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:0.0:1787826259.190978:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:0.0:1787826259.190979:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:0.0:1787826259.190981:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:0.0:1787826259.190982:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:0.0:1787826259.190984:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:0.0:1787826259.190986:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:0.0:1787826259.190987:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:0.0:1787826259.190988:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:0.0:1787826259.190990:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:0.0:1787826259.190991:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:0.0:1787826259.190993:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:0.0:1787826259.190994:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:0.0:1787826259.190996:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:0.0:1787826259.190998:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:0.0:1787826259.190999:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:0.0:1787826259.191001:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:0.0:1787826259.191003:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:0.0:1787826259.191004:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:0.0:1787826259.191006:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:0.0:1787826259.191008:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:0.0:1787826259.191009:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:0.0:1787826259.191011:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:0.0:1787826259.191012:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:0.0:1787826259.191013:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:0.0:1787826259.191015:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:0.0:1787826259.191017:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:0.0:1787826259.191018:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:0.0:1787826259.191020:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:0.0:1787826259.191022:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:0.0:1787826259.191023:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:0.0:1787826259.191024:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:0.0:1787826259.191026:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:0.0:1787826259.191028:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:0.0:1787826259.191029:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:0.0:1787826259.191031:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:0.0:1787826259.191032:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:0.0:1787826259.191033:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:0.0:1787826259.191035:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:0.0:1787826259.191037:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:0.0:1787826259.191038:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:0.0:1787826259.191040:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:0.0:1787826259.191041:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:0.0:1787826259.191043:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:0.0:1787826259.191044:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:0.0:1787826259.191046:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:0.0:1787826259.191048:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:0.0:1787826259.191049:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:0.0:1787826259.191051:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:0.0:1787826259.191052:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:0.0:1787826259.191053:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:0.0:1787826259.191055:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:0.0:1787826259.191056:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:0.0:1787826259.191057:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:0.0:1787826259.191059:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:0.0:1787826259.191060:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:0.0:1787826259.191061:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:0.0:1787826259.191063:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:0.0:1787826259.191065:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:0.0:1787826259.191066:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:0.0:1787826259.191068:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:0.0:1787826259.191070:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:0.0:1787826259.191071:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:0.0:1787826259.191073:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:0.0:1787826259.191075:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:0.0:1787826259.191086:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:0.0:1787826259.191089:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:0.0:1787826259.191090:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:0.0:1787826259.191092:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:0.0:1787826259.191094:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:0.0:1787826259.191095:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:0.0:1787826259.191097:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:0.0:1787826259.191099:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:0.0:1787826259.191101:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:0.0:1787826259.191102:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:0.0:1787826259.191104:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:0.0:1787826259.191106:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:0.0:1787826259.191107:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:0.0:1787826259.191109:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:0.0:1787826259.191111:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:0.0:1787826259.191112:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:0.0:1787826259.191114:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:0.0:1787826259.191115:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:0.0:1787826259.191117:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:0.0:1787826259.191118:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:0.0:1787826259.191120:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:0.0:1787826259.191122:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:0.0:1787826259.191123:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:0.0:1787826259.191125:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:0.0:1787826259.191127:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:0.0:1787826259.191129:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:0.0:1787826259.191131:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:0.0:1787826259.191132:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:0.0:1787826259.191134:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:0.0:1787826259.191135:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:0.0:1787826259.191137:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:0.0:1787826259.191138:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:0.0:1787826259.191149:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:0.0:1787826259.191150:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:0.0:1787826259.191152:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:0.0:1787826259.191154:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:0.0:1787826259.191156:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:0.0:1787826259.191158:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:0.0:1787826259.191159:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:0.0:1787826259.191162:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:0.0:1787826259.191163:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:0.0:1787826259.191165:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:0.0:1787826259.191167:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:0.0:1787826259.191169:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:0.0:1787826259.191170:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:0.0:1787826259.191172:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:0.0:1787826259.191174:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:0.0:1787826259.191175:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:0.0:1787826259.191177:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:0.0:1787826259.191179:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:0.0:1787826259.191181:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:0.0:1787826259.191183:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:0.0:1787826259.191184:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:0.0:1787826259.191186:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:0.0:1787826259.191188:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:0.0:1787826259.191189:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:0.0:1787826259.191191:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:0.0:1787826259.191193:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:0.0:1787826259.191195:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:0.0:1787826259.191196:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:0.0:1787826259.191198:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:0.0:1787826259.191199:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:0.0:1787826259.191201:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:0.0:1787826259.191202:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:0.0:1787826259.191204:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:0.0:1787826259.191206:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:0.0:1787826259.191208:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:0.0:1787826259.191210:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:0.0:1787826259.191212:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:0.0:1787826259.191214:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:0.0:1787826259.191216:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:0.0:1787826259.191218:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:0.0:1787826259.191220:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:0.0:1787826259.191221:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:0.0:1787826259.191223:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:0.0:1787826259.191224:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:0.0:1787826259.191226:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:0.0:1787826259.191228:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:0.0:1787826259.191229:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:0.0:1787826259.191231:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:0.0:1787826259.191232:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:0.0:1787826259.191234:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:0.0:1787826259.191236:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:0.0:1787826259.191237:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:0.0:1787826259.191239:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:0.0:1787826259.191240:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:0.0:1787826259.191241:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:0.0:1787826259.191243:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:0.0:1787826259.191245:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:0.0:1787826259.191247:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:0.0:1787826259.191248:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:0.0:1787826259.191250:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:0.0:1787826259.191252:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:0.0:1787826259.191253:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:0.0:1787826259.191255:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:0.0:1787826259.191257:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:0.0:1787826259.191258:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:0.0:1787826259.191260:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:0.0:1787826259.191261:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:0.0:1787826259.191263:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:0.0:1787826259.191265:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:0.0:1787826259.191267:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1787826259.191520:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1787826259.191543:0:2348:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907d908aad80 00000100:00100000:0.0:1787826259.191580:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011399040:192.168.203.104@tcp:4:lfs.0 00000080:00200000:1.0:1787826259.191813:0:65912:0:(file.c:2207:ll_file_io_generic()) Range unlock [0, 255] 00000080:00200000:1.0:1787826259.191826:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826259.191830:0:65912:0:(file.c:2240:ll_file_io_generic()) f204a.sanity-flr: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:1.0:1787826259.191841:0:65912:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000020:00200000:1.0:1787826259.191844:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826259.191854:0:65912:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200080:1.0:1787826259.191871:0:65912:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148) cmd=400866b1 arg=0 00000080:00200080:1.0:1787826259.191880:0:65912:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148) cmd=400866b1 arg=2 00000080:00200008:1.0:1787826259.191887:0:65912:0:(file.c:6313:ll_getattr_dentry()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0](ffff907d92219148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826259.191894:0:65912:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148),name=f204a.sanity-flr 00000002:00010000:1.0:1787826259.191911:0:65912:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1fd:0x0]) in obj [0x200000404:0x1fd:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826259.191930:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826259.191944:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1787826259.191953:0:65912:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1fd:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826259.191960:0:65912:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) 00000080:00010000:1.0:1787826259.191964:0:65912:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) for lock 0xe5f736940357b6dd 00000080:00010000:1.0:1787826259.191971:0:65912:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b6dd from it ffff9dd14195bcd8 00010000:00010000:1.0:1787826259.191975:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826259.191985:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1787826259.192006:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:1.0:1787826259.192017:0:65912:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826259.192020:0:65912:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826259.192025:0:65912:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1fd:0x0] 00000008:00010000:1.0:1787826259.192041:0:65912:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826259.192044:0:65912:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c326b0, osc lock ffff907d962f5248, flags 0x400000400001002 00010000:00010000:1.0:1787826259.192059:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826259.192071:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00200000:1.0:1787826259.192093:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826259.192097:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826259.192106:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000020:00200000:1.0:1787826259.192115:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826259.192119:0:65912:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:1.0:1787826259.192123:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_release():245 00000020:00010000:1.0:1787826259.192126:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826259.192129:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c326b0 (ffff907da56082d8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826259.192134:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826259.192137:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c326b0 (ffff907da5608368/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826259.192248:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826259.192260:0:65912:0:(file.c:2652:do_file_write_iter()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0], ppos: 1048576, count: 1048576 00000080:00200000:1.0:1787826259.192266:0:65912:0:(file.c:2090:ll_file_io_generic()) f204a.sanity-flr: write ppos: 1048576, bytes: 1048576 00000080:00200000:1.0:1787826259.192273:0:65912:0:(file.c:1848:ll_io_set_mirror()) f204a.sanity-flr: desiginated mirror: 2 00000020:00200000:1.0:1787826259.192279:0:65912:0:(cl_io.c:218:cl_io_rw_init()) header@000000007d7e8202[0x110, 2, [0x200000404:0x1fd:0x0] hash] 00000020:00200000:1.0:1787826259.192281:0:65912:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1048576, 2097152) 2048 0 00000080:00200000:1.0:1787826259.192284:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00000080:00200000:1.0:1787826259.192293:0:65912:0:(file.c:2170:ll_file_io_generic()) Range lock [256, 511] 00020000:00200000:1.0:1787826259.192300:0:65912:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 1048576 chunk: [1048576, 2097152) 2097152, 1048576 00020000:00200000:1.0:1787826259.192303:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[3] flags 0x11 00000080:00200000:1.0:1787826259.192313:0:65912:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [256, 511] 00000008:00010000:1.0:1787826259.192321:0:65912:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 11 00000008:00010000:1.0:1787826259.192324:0:65912:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c326b0, osc lock ffff907d962f5248, flags 0x40000 00000080:00200000:1.0:1787826259.192331:0:65912:0:(vvp_io.c:1339:vvp_io_write_start()) f204a.sanity-flr: write [1048576, 2097152) append 0 append lockpos 0 00000080:00200000:1.0:1787826259.192343:0:65912:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148), size=1048576 (max 335544320), offset=1048576=0x100000, pages 256 (max 81920), lockless, parallel 00000020:00200000:1.0:1787826259.192922:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:1.0:1787826259.192925:0:65912:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 2097152 00000080:00200000:1.0:1787826259.192934:0:65912:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:1.0:1787826259.192938:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d80024c68 (ffff907da5608368/0) at cl_lock_release():245 00000020:00010000:1.0:1787826259.192941:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d80024c68 (ffff907da5608368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826259.192944:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c326b0 (ffff907da56082d8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826259.192947:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d80024c68 (ffff907da5608368/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826259.192949:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c326b0 (ffff907da5608368/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826259.192955:0:65912:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:1.0:1787826259.192961:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1787826259.192978:0:49858:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907d908aac00 00000008:00100000:1.0:1787826259.194805:0:49858:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d908a30c0 - object 0x280000400:776 offset 1048576<>2097152 00000100:00100000:0.0:1787826259.194876:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011399168:192.168.203.104@tcp:4:lfs.0 00000100:00100000:0.0:1787826259.229900:0:2349:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0001-osc-ffff907d986d0000: committing for last_committed 17179873821 gen 1 00000008:00010000:0.0:1787826259.229951:0:2349:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907d908aac00 00000020:00200000:0.0:1787826259.229958:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:0.0:1787826259.229960:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:0.0:1787826259.229962:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:0.0:1787826259.229964:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:0.0:1787826259.229965:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:0.0:1787826259.229967:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:0.0:1787826259.229968:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:0.0:1787826259.229970:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:0.0:1787826259.229972:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:0.0:1787826259.229974:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:0.0:1787826259.229975:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:0.0:1787826259.229978:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:0.0:1787826259.229980:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:0.0:1787826259.229981:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:0.0:1787826259.229983:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:0.0:1787826259.229984:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:0.0:1787826259.229986:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:0.0:1787826259.229988:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:0.0:1787826259.229989:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:0.0:1787826259.229990:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:0.0:1787826259.229992:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:0.0:1787826259.229993:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:0.0:1787826259.229995:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:0.0:1787826259.229997:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:0.0:1787826259.229999:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:0.0:1787826259.230000:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:0.0:1787826259.230002:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:0.0:1787826259.230004:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:0.0:1787826259.230005:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:0.0:1787826259.230007:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:0.0:1787826259.230008:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:0.0:1787826259.230010:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:0.0:1787826259.230011:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:0.0:1787826259.230013:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:0.0:1787826259.230015:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:0.0:1787826259.230017:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:0.0:1787826259.230019:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:0.0:1787826259.230020:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:0.0:1787826259.230022:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:0.0:1787826259.230024:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:0.0:1787826259.230026:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:0.0:1787826259.230027:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:0.0:1787826259.230029:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:0.0:1787826259.230031:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:0.0:1787826259.230032:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:0.0:1787826259.230034:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:0.0:1787826259.230035:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:0.0:1787826259.230037:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:0.0:1787826259.230038:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:0.0:1787826259.230040:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:0.0:1787826259.230041:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:0.0:1787826259.230042:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:0.0:1787826259.230044:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:0.0:1787826259.230046:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:0.0:1787826259.230047:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:0.0:1787826259.230049:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:0.0:1787826259.230051:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:0.0:1787826259.230052:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:0.0:1787826259.230054:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:0.0:1787826259.230055:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:0.0:1787826259.230056:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:0.0:1787826259.230058:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:0.0:1787826259.230059:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:0.0:1787826259.230061:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:0.0:1787826259.230063:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:0.0:1787826259.230064:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:0.0:1787826259.230066:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:0.0:1787826259.230067:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:0.0:1787826259.230068:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:0.0:1787826259.230070:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:0.0:1787826259.230071:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:0.0:1787826259.230073:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:0.0:1787826259.230075:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:0.0:1787826259.230076:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:0.0:1787826259.230078:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:0.0:1787826259.230080:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:0.0:1787826259.230082:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:0.0:1787826259.230083:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:0.0:1787826259.230085:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:0.0:1787826259.230086:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:0.0:1787826259.230088:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:0.0:1787826259.230089:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:0.0:1787826259.230091:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:0.0:1787826259.230093:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:0.0:1787826259.230094:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:0.0:1787826259.230096:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:0.0:1787826259.230098:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:0.0:1787826259.230099:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:0.0:1787826259.230101:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:0.0:1787826259.230103:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:0.0:1787826259.230104:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:0.0:1787826259.230106:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:0.0:1787826259.230107:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:0.0:1787826259.230108:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:0.0:1787826259.230110:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:0.0:1787826259.230111:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:0.0:1787826259.230113:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:0.0:1787826259.230114:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:0.0:1787826259.230116:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:0.0:1787826259.230117:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:0.0:1787826259.230119:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:0.0:1787826259.230120:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:0.0:1787826259.230121:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:0.0:1787826259.230123:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:0.0:1787826259.230124:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:0.0:1787826259.230126:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:0.0:1787826259.230127:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:0.0:1787826259.230129:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:0.0:1787826259.230131:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:0.0:1787826259.230132:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:0.0:1787826259.230134:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:0.0:1787826259.230135:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:0.0:1787826259.230136:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:0.0:1787826259.230151:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:0.0:1787826259.230153:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:0.0:1787826259.230154:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:0.0:1787826259.230155:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:0.0:1787826259.230157:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:0.0:1787826259.230159:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:0.0:1787826259.230160:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:0.0:1787826259.230162:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:0.0:1787826259.230163:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:0.0:1787826259.230164:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:0.0:1787826259.230166:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:0.0:1787826259.230167:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:0.0:1787826259.230169:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:0.0:1787826259.230170:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:0.0:1787826259.230171:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:0.0:1787826259.230172:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:0.0:1787826259.230174:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:0.0:1787826259.230175:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:0.0:1787826259.230177:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:0.0:1787826259.230178:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:0.0:1787826259.230179:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:0.0:1787826259.230181:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:0.0:1787826259.230182:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:0.0:1787826259.230183:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:0.0:1787826259.230184:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:0.0:1787826259.230186:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:0.0:1787826259.230188:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:0.0:1787826259.230189:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:0.0:1787826259.230191:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:0.0:1787826259.230192:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:0.0:1787826259.230194:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:0.0:1787826259.230195:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:0.0:1787826259.230196:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:0.0:1787826259.230197:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:0.0:1787826259.230198:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:0.0:1787826259.230200:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:0.0:1787826259.230201:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:0.0:1787826259.230202:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:0.0:1787826259.230203:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:0.0:1787826259.230205:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:0.0:1787826259.230206:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:0.0:1787826259.230208:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:0.0:1787826259.230209:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:0.0:1787826259.230210:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:0.0:1787826259.230212:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:0.0:1787826259.230213:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:0.0:1787826259.230215:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:0.0:1787826259.230216:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:0.0:1787826259.230217:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:0.0:1787826259.230219:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:0.0:1787826259.230221:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:0.0:1787826259.230222:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:0.0:1787826259.230224:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:0.0:1787826259.230225:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:0.0:1787826259.230226:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:0.0:1787826259.230228:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:0.0:1787826259.230229:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:0.0:1787826259.230230:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:0.0:1787826259.230232:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:0.0:1787826259.230234:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:0.0:1787826259.230235:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:0.0:1787826259.230236:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:0.0:1787826259.230238:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:0.0:1787826259.230239:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:0.0:1787826259.230241:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:0.0:1787826259.230242:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:0.0:1787826259.230243:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:0.0:1787826259.230245:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:0.0:1787826259.230246:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:0.0:1787826259.230248:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:0.0:1787826259.230249:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:0.0:1787826259.230251:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:0.0:1787826259.230253:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:0.0:1787826259.230255:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:0.0:1787826259.230256:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:0.0:1787826259.230258:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:0.0:1787826259.230259:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:0.0:1787826259.230261:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:0.0:1787826259.230263:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:0.0:1787826259.230264:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:0.0:1787826259.230265:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:0.0:1787826259.230266:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:0.0:1787826259.230268:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:0.0:1787826259.230269:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:0.0:1787826259.230271:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:0.0:1787826259.230272:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:0.0:1787826259.230273:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:0.0:1787826259.230275:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:0.0:1787826259.230276:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:0.0:1787826259.230277:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:0.0:1787826259.230279:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:0.0:1787826259.230280:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:0.0:1787826259.230282:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:0.0:1787826259.230283:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:0.0:1787826259.230284:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:0.0:1787826259.230286:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:0.0:1787826259.230287:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:0.0:1787826259.230288:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:0.0:1787826259.230289:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:0.0:1787826259.230291:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:0.0:1787826259.230292:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:0.0:1787826259.230294:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:0.0:1787826259.230295:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:0.0:1787826259.230296:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:0.0:1787826259.230298:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:0.0:1787826259.230300:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:0.0:1787826259.230301:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:0.0:1787826259.230303:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:0.0:1787826259.230304:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:0.0:1787826259.230306:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:0.0:1787826259.230307:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:0.0:1787826259.230308:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:0.0:1787826259.230309:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:0.0:1787826259.230311:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:0.0:1787826259.230313:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:0.0:1787826259.230314:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:0.0:1787826259.230315:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:0.0:1787826259.230316:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:0.0:1787826259.230318:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:0.0:1787826259.230319:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:0.0:1787826259.230320:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:0.0:1787826259.230321:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:0.0:1787826259.230323:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:0.0:1787826259.230325:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:0.0:1787826259.230327:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:0.0:1787826259.230328:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:0.0:1787826259.230329:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:0.0:1787826259.230331:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:0.0:1787826259.230332:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:0.0:1787826259.230333:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:0.0:1787826259.230334:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:0.0:1787826259.230336:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:0.0:1787826259.230338:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:0.0:1787826259.230339:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:0.0:1787826259.230340:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:0.0:1787826259.230342:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:0.0:1787826259.230343:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:0.0:1787826259.230344:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:0.0:1787826259.230345:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:0.0:1787826259.230347:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:0.0:1787826259.230348:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:0.0:1787826259.230350:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:0.0:1787826259.230351:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1787826259.230709:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1787826259.230733:0:2349:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907d908aac00 00000100:00100000:0.0:1787826259.230759:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011399168:192.168.203.104@tcp:4:lfs.0 00000080:00200000:2.0:1787826259.230967:0:65912:0:(file.c:2207:ll_file_io_generic()) Range unlock [256, 511] 00000080:00200000:2.0:1787826259.230980:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826259.230985:0:65912:0:(file.c:2240:ll_file_io_generic()) f204a.sanity-flr: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1787826259.230994:0:65912:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000020:00200000:2.0:1787826259.230998:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826259.231006:0:65912:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200080:2.0:1787826259.231023:0:65912:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148) cmd=400866b1 arg=0 00000080:00200080:2.0:1787826259.231030:0:65912:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148) cmd=400866b1 arg=1 00000080:00200008:2.0:1787826259.231036:0:65912:0:(file.c:6313:ll_getattr_dentry()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0](ffff907d92219148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1787826259.231043:0:65912:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148),name=f204a.sanity-flr 00000002:00010000:2.0:1787826259.231060:0:65912:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1fd:0x0]) in obj [0x200000404:0x1fd:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826259.231078:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826259.231091:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1787826259.231098:0:65912:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1fd:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826259.231106:0:65912:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) 00000080:00010000:2.0:1787826259.231109:0:65912:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) for lock 0xe5f736940357b6dd 00000080:00010000:2.0:1787826259.231116:0:65912:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b6dd from it ffff9dd14195bcd8 00010000:00010000:2.0:1787826259.231120:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826259.231129:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1787826259.231160:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:2.0:1787826259.231170:0:65912:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826259.231173:0:65912:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1787826259.231177:0:65912:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1fd:0x0] 00000008:00010000:2.0:1787826259.231194:0:65912:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826259.231197:0:65912:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322cb0, osc lock ffff907d830a3380, flags 0x400000400001002 00010000:00010000:2.0:1787826259.231214:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1787826259.231225:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000020:00200000:2.0:1787826259.231234:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826259.231237:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:2.0:1787826259.231244:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000020:00200000:2.0:1787826259.231252:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826259.231256:0:65912:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826259.231260:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_release():245 00000020:00010000:2.0:1787826259.231263:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826259.231265:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322cb0 (ffff907da56082d8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826259.231268:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826259.231271:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322cb0 (ffff907da5608368/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826259.231277:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826259.231289:0:65912:0:(file.c:5224:ll_file_seek()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0], offset: 2097152, type: SEEK_DATA 00000080:00200000:2.0:1787826259.231293:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:2.0:1787826259.231297:0:65912:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826259.231299:0:65912:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1787826259.231303:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1787826259.231307:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:2.0:1787826259.231319:0:65912:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000080:00200000:2.0:1787826259.231326:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826259.231329:0:65912:0:(file.c:1848:ll_io_set_mirror()) f204a.sanity-flr: desiginated mirror: 1 00000080:00200000:2.0:1787826259.231331:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1787826259.231336:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1787826259.231338:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:2.0:1787826259.231340:0:65912:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [512, 18446744073709551615] 00000008:00010000:2.0:1787826259.231346:0:65912:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:2.0:1787826259.231348:0:65912:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322cb0, osc lock ffff907d830a3380, flags 0x0 00010000:00010000:2.0:1787826259.231355:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1787826259.231374:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (2097152 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1787826259.231383:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1787826259.231395:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1787826259.231401:0:65912:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000080:00200000:2.0:1787826259.231410:0:65912:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826259.231414:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d80024c68 (ffff907da5608368/0) at cl_lock_release():245 00000020:00010000:2.0:1787826259.231416:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d80024c68 (ffff907da5608368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826259.231418:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322cb0 (ffff907da56082d8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1787826259.231421:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1787826259.231429:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00000020:00010000:2.0:1787826259.231436:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d80024c68 (ffff907da5608368/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826259.231438:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322cb0 (ffff907da5608368/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826259.231443:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826259.231446:0:65912:0:(file.c:5262:ll_file_seek()) COMPLETED file f204a.sanity-flr:[0x200000404:0x1fd:0x0], offset: 2097152, type: SEEK_DATA, rc = -6 00000080:00200008:2.0:1787826259.231451:0:65912:0:(file.c:5224:ll_file_seek()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0], offset: 0, type: (null) 00000080:00200000:2.0:1787826259.231455:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:2.0:1787826259.231458:0:65912:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826259.231461:0:65912:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1787826259.231463:0:65912:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1fd:0x0] 00000008:00010000:2.0:1787826259.231468:0:65912:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826259.231470:0:65912:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322cb0, osc lock ffff907d830a3380, flags 0x400000400001002 00010000:00010000:2.0:1787826259.231477:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:2.0:1787826259.231484:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00000020:00200000:2.0:1787826259.231492:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826259.231496:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:2.0:1787826259.231502:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00000020:00200000:2.0:1787826259.231510:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826259.231513:0:65912:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826259.231515:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_release():245 00000020:00010000:2.0:1787826259.231518:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826259.231520:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322cb0 (ffff907da56082d8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826259.231523:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826259.231526:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322cb0 (ffff907da5608368/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826259.231532:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826259.231536:0:65912:0:(file.c:5262:ll_file_seek()) COMPLETED file f204a.sanity-flr:[0x200000404:0x1fd:0x0], offset: 0, type: (null), rc = 2097152 00000080:00200080:2.0:1787826259.231541:0:65912:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148) cmd=400866b1 arg=0 00000080:00200080:2.0:1787826259.245547:0:65912:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148) cmd=400866b1 arg=2 00000080:00200008:2.0:1787826259.245580:0:65912:0:(file.c:6313:ll_getattr_dentry()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0](ffff907d92219148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1787826259.245590:0:65912:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148),name=f204a.sanity-flr 00000002:00010000:2.0:1787826259.245695:0:65912:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1fd:0x0]) in obj [0x200000404:0x1fd:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826259.245714:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1787826259.245728:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1787826259.245733:0:65912:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1fd:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826259.245739:0:65912:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) 00000080:00010000:2.0:1787826259.245742:0:65912:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) for lock 0xe5f736940357b6dd 00000080:00010000:2.0:1787826259.245748:0:65912:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b6dd from it ffff9dd14195bcd8 00010000:00010000:2.0:1787826259.245751:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1787826259.245757:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1787826259.245780:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:2.0:1787826259.245791:0:65912:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826259.245794:0:65912:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1787826259.245798:0:65912:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x1fd:0x0] 00000008:00010000:2.0:1787826259.246244:0:65912:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826259.246249:0:65912:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322cb0, osc lock ffff907d830a3380, flags 0x400000400001002 00010000:00010000:2.0:1787826259.246267:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1787826259.246279:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00000020:00200000:2.0:1787826259.246291:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826259.246295:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1787826259.246303:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00000020:00200000:2.0:1787826259.246315:0:65912:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826259.246325:0:65912:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826259.246329:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_release():245 00000020:00010000:2.0:1787826259.246332:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826259.246335:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322cb0 (ffff907da56082d8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826259.246338:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98afc000 (ffff907da5608368/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826259.246341:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322cb0 (ffff907da5608368/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826259.246353:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826259.246382:0:65912:0:(llite_lib.c:2427:ll_setattr_raw()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0](ffff907d92219148) current size 2097152, valid attrs 2068, mode 0, uid -1976721614, gid -1, new size 2097152, atime 1787826259.000000000, mtime 1787826259.000000000, ctime 1787826259.000000000 00010000:00010000:2.0:1787826259.246405:0:65912:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1787826259.246421:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826259.247218:0:65912:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 1/0,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1787826259.247230:0:65912:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1787826259.247233:0:65912:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357b6dd lrc: 0/0,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0f830 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00000020:00100000:2.0:1787826259.247253:0:65912:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1787826259.247264:0:65912:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114a940 x1874670011399424/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787826259.247286:0:65912:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011399424:192.168.203.104@tcp:36:lfs.0 00000100:00100000:2.0:1787826259.247375:0:65912:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6480 going to sleep for 13 seconds 00000100:00100000:0.0:1787826259.256733:0:65912:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011399424:192.168.203.104@tcp:36:lfs.0 00000080:00200000:0.0:1787826259.256773:0:65912:0:(file.c:1848:ll_io_set_mirror()) f204a.sanity-flr: desiginated mirror: 2 00000080:00200000:0.0:1787826259.256778:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 6 restore needed 0 00020000:00200000:0.0:1787826259.256806:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[3] flags 0x11 00020000:00200000:0.0:1787826259.256823:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[4] flags 0x40 00000080:00200000:0.0:1787826259.256826:0:65912:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:0.0:1787826259.256843:0:65912:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1787826259.256847:0:65912:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d888674b0, osc lock ffff907d96368618, flags 0x0 00000008:00010000:0.0:1787826259.256851:0:65912:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1787826259.256852:0:65912:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d888674f0, osc lock ffff907d96369728, flags 0x0 00010000:00010000:0.0:1787826259.256899:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b6eb lrc: 3/0,1 mode: --/PW res: [0x280000400:0x307:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.256911:0:65912:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b6eb lrc: 3/0,1 mode: --/PW res: [0x280000400:0x307:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.256921:0:65912:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b6eb lrc: 3/0,1 mode: --/PW res: [0x280000400:0x307:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826259.256939:0:65912:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011399552:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826259.257001:0:65912:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d888670c0 going to sleep for 11 seconds 00000100:00100000:0.0:1787826259.261933:0:65912:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011399552:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826259.261960:0:65912:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b6eb lrc: 4/0,1 mode: --/PW res: [0x280000400:0x307:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f85a expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.261978:0:65912:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b6eb lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x307:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f85a expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.261989:0:65912:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b6eb lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x307:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f85a expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.262020:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b6eb lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x307:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f85a expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826259.262032:0:65912:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 1048576, setting rss=1048576; kms=1048576, end=18446744073709551615 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b6eb lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x307:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f85a expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.262045:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b6eb lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x307:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f85a expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.262054:0:65912:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b6eb lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x307:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f85a expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.262114:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b6f2 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x308:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.262123:0:65912:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b6f2 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x308:0x0].0x0 rrc: 2 type: EXT [1048576->68157439] (req 1048576->68157439) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.262133:0:65912:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b6f2 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x308:0x0].0x0 rrc: 2 type: EXT [1048576->68157439] (req 1048576->68157439) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826259.262152:0:65912:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011399680:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826259.262226:0:65912:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d888670c0 going to sleep for 11 seconds 00000100:00100000:0.0:1787826259.266599:0:65912:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011399680:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826259.266666:0:65912:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b6f2 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x308:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f861 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.266685:0:65912:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b6f2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x308:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f861 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.266704:0:65912:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b6f2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x308:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f861 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.266719:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b6f2 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x308:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f861 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826259.266743:0:65912:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b6f2 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x308:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f861 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.266756:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b6f2 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x308:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f861 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.266765:0:65912:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b6f2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x308:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f861 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826259.267204:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1787826259.267218:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011399808:192.168.203.104@tcp:10:lfs.0 00000100:00100000:0.0:1787826259.275588:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011399808:192.168.203.104@tcp:10:lfs.0 00000080:00200000:2.0:1787826259.275809:0:65912:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x1fd:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826259.275816:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b1c68 (ffff907d884a7998/0) at cl_lock_release():245 00000020:00010000:2.0:1787826259.275819:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b1c68 (ffff907d884a7998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826259.275824:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d888674b0 (ffff907d884a7638/1) at cl_lock_cancel():142 00010000:00010000:2.0:1787826259.275834:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b6eb lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x307:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f85a expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.275849:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b6eb lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x307:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f85a expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1787826259.275864:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d888674f0 (ffff907d884a7488/1) at cl_lock_cancel():142 00010000:00010000:2.0:1787826259.275869:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b6f2 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x308:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0f861 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826259.275877:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b6f2 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x308:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f861 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1787826259.275891:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b1c68 (ffff907d884a7998/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826259.275894:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d888674b0 (ffff907d884a7998/1) at cl_lock_fini():72 00000020:00010000:2.0:1787826259.275899:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d888674f0 (ffff907d884a7998/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826259.275917:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826259.275929:0:65912:0:(llite_lib.c:2686:ll_setattr_raw()) COMPLETED file f204a.sanity-flr:[0x200000404:0x1fd:0x0](ffff907d92219148) current size 2097152, valid attrs 10002168, mode 0, uid -1976721614, gid -1, new size 2097152, atime 1787826259.000000000, mtime 1787826259.000000000, ctime 1787826259.000000000, rc 0 00000080:00200080:2.0:1787826259.275948:0:65912:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148) cmd=400866b1 arg=0 00000080:00200008:2.0:1787826259.275986:0:65912:0:(llite_lib.c:2427:ll_setattr_raw()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0](ffff907d92219148) current size 2097152, valid attrs 101f0, mode 9dd1, uid -1828189880, gid -28547, new size 18446621467729826120, atime 1787826258.000000000, mtime 1787826258.000000000, ctime 1787826259.000000000 00000020:00100000:2.0:1787826259.276157:0:65912:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1787826259.276167:0:65912:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114bc00 x1874670011400064/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787826259.276188:0:65912:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011400064:192.168.203.104@tcp:36:lfs.0 00000100:00100000:2.0:1787826259.276340:0:65912:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f66c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826259.289049:0:65912:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011400064:192.168.203.104@tcp:36:lfs.0 00000080:00200000:0.0:1787826259.289094:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826259.289110:0:65912:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826259.289113:0:65912:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826259.289120:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826259.289168:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:0.0:1787826259.289173:0:65912:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:0.0:1787826259.289185:0:65912:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1787826259.289189:0:65912:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da032edb0, osc lock ffff907d96369248, flags 0x0 00010000:00010000:0.0:1787826259.289204:0:65912:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d91e6e400 type 11 mode 2 res 9663677440/803 (0 10485759) 00010000:00010000:0.0:1787826259.289256:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.289269:0:65912:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.289278:0:65912:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826259.289303:0:65912:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011400192:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826259.297089:0:65912:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d888670c0 going to sleep for 11 seconds 00000100:00100000:2.0:1787826259.303714:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826259.303832:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025481216 00000100:00100000:3.0:1787826259.303841:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025481216 00000100:00100000:3.0:1787826259.303897:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7127 00000100:00100000:3.0:1787826259.303975:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91148b40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:36727:x1874670025481216:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826259.303992:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00010000:00010000:3.0:1787826259.304071:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826259.304077:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91148b40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:36727:x1874670025481216:12345-192.168.203.104@tcp:104: Request processed in 168us (372us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826259.304085:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7127 00010000:00010000:0.0:1787826259.304338:0:61297:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826259.304346:0:61297:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00010000:00010000:0.0:1787826259.304366:0:61297:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0ba00 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826259.304372:0:61297:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800428400000000 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00000080:00200000:0.0:1787826259.304444:0:61297:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1787826259.304457:0:61297:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1787826259.304500:0:61297:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 2/0,0 mode: --/PR res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00010000:00010000:0.0:1787826259.304510:0:61297:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826259.304537:0:61297:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00010000:00010000:0.0:1787826259.304545:0:61297:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000001ce8168e/0xe5f736940357b6e4 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0f837 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00000100:00100000:1.0:1787826259.304563:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011400320:192.168.203.104@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1787826259.304636:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826259.304639:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826259.304641:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826259.309153:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:0.0:1787826259.309329:0:65912:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011400192:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826259.309357:0:65912:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.309380:0:65912:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.309388:0:65912:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826259.309555:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025481344 00000100:00100000:3.0:1787826259.309562:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025481344 00000100:00100000:3.0:1787826259.309658:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7128 00000100:00100000:3.0:1787826259.309667:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d9114ad00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:8014:x1874670025481344:12345-192.168.203.104@tcp:105: 00010000:00010000:3.0:1787826259.309676:0:50445:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 5/0,1 mode: --/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826259.309684:0:50445:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 5/0,1 mode: --/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826259.309693:0:50445:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 5/0,1 mode: --/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826259.309701:0:50445:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10080000020000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826259.309715:0:50445:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10080000020000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826259.309848:0:50445:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x80000020000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826259.309879:0:50445:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff907d8883b000) 00010000:00010000:0.0:1787826259.309995:0:65912:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.310005:0:65912:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.310026:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826259.310039:0:65912:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10020000020000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.310053:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.310061:0:65912:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826259.310202:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1787826259.310217:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011400448:192.168.203.104@tcp:2:lfs.0 00000100:00100000:3.0:1787826259.311054:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d9114ad00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:8014:x1874670025481344:12345-192.168.203.104@tcp:105: Request processed in 1384us (1915us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826259.311064:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7128 00000100:00100000:1.0:1787826259.317582:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011400448:192.168.203.104@tcp:2:lfs.0 00000100:00100000:1.0:1787826259.317604:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011400320:192.168.203.104@tcp:103:ldlm_bl.0 00000020:00010000:0.0:1787826259.317828:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b1038 (ffff907d884a7878/0) at cl_lock_release():245 00000020:00010000:0.0:1787826259.317832:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b1038 (ffff907d884a7878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1787826259.317838:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da032edb0 (ffff907d884a7b48/1) at cl_lock_cancel():142 00010000:00010000:0.0:1787826259.317848:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.317863:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:0.0:1787826259.317879:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b1038 (ffff907d884a7878/0) at cl_lock_fini():72 00000020:00010000:0.0:1787826259.317882:0:65912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da032edb0 (ffff907d884a7878/1) at cl_lock_fini():72 00000080:00200000:0.0:1787826259.317899:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1787826259.317980:0:65912:0:(llite_lib.c:2686:ll_setattr_raw()) COMPLETED file f204a.sanity-flr:[0x200000404:0x1fd:0x0](ffff907d92219148) current size 2097152, valid attrs 100101f0, mode 9dd1, uid -1828189880, gid -28547, new size 18446621467729826120, atime 1787826258.000000000, mtime 1787826258.000000000, ctime 1787826259.000000000, rc 0 00000080:00200080:0.0:1787826259.318003:0:65912:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148) cmd=c00c66f3 arg=aed2a0 00010000:00010000:0.0:1787826259.318051:0:65912:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b6cf lrc: 2/0,0 mode: PW/PW res: [0x200000404:0x1fd:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x81008000000000 nid: local remote: 0x7918213f4fc0f806 expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.318068:0:65912:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b6cf lrc: 0/0,0 mode: --/PW res: [0x200000404:0x1fd:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x85809400000000 nid: local remote: 0x7918213f4fc0f806 expref: -99 pid: 65912 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826259.318087:0:65912:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d908a12c0 x1874670011397632/t8589942945(8589942945) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826277 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1787826259.318098:0:65912:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1787826259.318103:0:65912:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90847840 x1874670011400704/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 504/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1787826259.318123:0:65912:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011400704:192.168.203.104@tcp:35:lfs.0 00000100:00100000:0.0:1787826259.318311:0:65912:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d888670c0 going to sleep for 11 seconds 00000100:00100000:2.0:1787826259.325944:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826259.326122:0:40339:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:3.0:1787826259.326150:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025481472 00000100:00100000:3.0:1787826259.326158:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025481472 00000100:00100000:3.0:1787826259.326211:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7129 00000100:00100000:3.0:1787826259.326231:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83fafc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:33311:x1874670025481472:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826259.326250:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1787826259.326321:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826259.326329:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83fafc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:33311:x1874670025481472:12345-192.168.203.104@tcp:104: Request processed in 110us (394us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826259.326336:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7129 00010000:00010000:1.0:1787826259.326433:0:47852:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826259.326440:0:47852:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826259.326449:0:47852:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0a400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826259.326454:0:47852:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826259.326463:0:47852:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826259.326471:0:47852:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 3/0,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1787826259.326494:0:47852:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x1fd:0x0]: losing layout lock 00010000:00010000:1.0:1787826259.326532:0:47852:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 2/0,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826259.326539:0:47852:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826259.326575:0:47852:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 1/0,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826259.326583:0:47852:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b6d6 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0f822 expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826259.326603:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826259.326604:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826259.326628:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826259.326636:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1787826259.326649:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011400832:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826259.332778:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011400832:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826259.346251:0:65912:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589942950 gen 1 00000100:00100000:0.0:1787826259.346281:0:65912:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011400704:192.168.203.104@tcp:35:lfs.0 00000002:00100000:0.0:1787826259.346310:0:65912:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d908a12c0 x1874670011397632/t8589942945(8589942945) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826277 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:0.0:1787826259.346340:0:65912:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 31 res 8589935620/509 (0 0) 00000080:00010000:0.0:1787826259.346350:0:65912:0:(file.c:7076:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000404:0x1fd:0x0](ffff907d92219148) 00000002:00010000:0.0:1787826259.346366:0:65912:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1fd:0x0]) in obj [0x200000404:0x1fd:0x0], intent: layout flags 0 00010000:00010000:0.0:1787826259.346408:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65912 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.346418:0:65912:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65912 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.346426:0:65912:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65912 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826259.346443:0:65912:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011400960:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826259.346526:0:65912:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d888670c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826259.353819:0:65912:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011400960:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826259.353850:0:65912:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f87d expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.353865:0:65912:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f87d expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826259.353878:0:65912:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x400 disposition=0x1, status=0 req@ffff907d90847840 x1874670011400960/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/896 e 0 to 0 dl 1787826277 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826259.353888:0:65912:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f87d expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1787826259.353899:0:65912:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f87d expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.353922:0:65912:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/509 (0 0) 00000080:00200000:0.0:1787826259.353929:0:65912:0:(namei.c:952:ll_intent_lock()) intent lock 1024 on i1 [0x200000404:0x1fd:0x0] suppgids 0 0: rc 0 00000080:00010000:0.0:1787826259.353943:0:65912:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) for lock 0xe5f736940357b700 00000080:00010000:0.0:1787826259.353952:0:65912:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b700 from it ffff9dd14195bbc8 00010000:00010000:0.0:1787826259.353955:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f87d expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.353964:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f87d expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826259.353984:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f87d expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826259.353993:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f87d expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:0.0:1787826259.354000:0:65912:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x1fd:0x0](00000000a294d51b) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f87d expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826259.354038:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f87d expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826259.354044:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f87d expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826259.354059:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826259.354068:0:65912:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826259.354071:0:65912:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826259.354076:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826259.354084:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:0.0:1787826259.354101:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1787826259.354124:0:65912:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f87d expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826259.354132:0:65912:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f87d expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1787826259.354138:0:65912:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x1fd:0x0](00000000a294d51b) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f87d expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826259.354155:0:65912:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826259.354160:0:65912:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826259.354162:0:65912:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826259.354164:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826259.354167:0:65912:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:0.0:1787826259.354173:0:65912:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:0.0:1787826259.354185:0:65912:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 2097152 blocks 4096 [cma]time: 1787826258 1787826258 0 by oinfo size 2097152 blocks 4096 [cma]time 1787826259 1787826258 1787826258 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x323:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:0.0:1787826259.354213:0:65912:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 1048576 blocks 2048 [cma]time: 1787826259 1787826259 0 by oinfo size 1048576 blocks 2048 [cma]time 1787826259 1787826259 0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b6eb lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x307:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f85a expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:0.0:1787826259.354238:0:65912:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 2097152 blocks 2048 [cma]time: 1787826259 1787826259 0 by oinfo size 2097152 blocks 2048 [cma]time 1787826259 1787826259 0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b6f2 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x308:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0f861 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826259.354290:0:65912:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1fd:0x0]: layout version change: 4294967294 -> 8 00010000:00010000:0.0:1787826259.354296:0:65912:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f87d expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826259.354308:0:65912:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f87d expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:0.0:1787826259.354338:0:65912:0:(file.c:479:ll_file_release()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0](ffff907d92219148), flags 154002 00000002:00100000:0.0:1787826259.354378:0:65912:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d90846d00 x1874670011397248/t8589942944(8589942944) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826277 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1787826259.354388:0:65912:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1787826259.354394:0:65912:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90847840 x1874670011401216/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1787826259.354414:0:65912:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011401216:192.168.203.104@tcp:35:lfs.0 00000100:00100000:0.0:1787826259.354898:0:65912:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d888670c0 going to sleep for 11 seconds 00000100:00100000:0.0:1787826259.365321:0:65912:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65912:1874670011401216:192.168.203.104@tcp:35:lfs.0 00000002:00100000:0.0:1787826259.365352:0:65912:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d90846d00 x1874670011397248/t8589942944(8589942944) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826277 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1787826259.388187:0:65914:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1787826259.388198:0:65914:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826259.388226:0:65914:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826259.388245:0:65914:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:2.0:1787826259.388261:0:65914:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:2.0:1787826259.388270:0:65914:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826259.388277:0:65914:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826259.388281:0:65914:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:2.0:1787826259.388287:0:65914:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd140d7fcb8 00010000:00010000:2.0:1787826259.388291:0:65914:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:2.0:1787826259.388300:0:65914:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:2.0:1787826259.388834:0:65914:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826259.388844:0:65914:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826259.388853:0:65914:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:2.0:1787826259.388863:0:65914:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:2.0:1787826259.388870:0:65914:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826259.388874:0:65914:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826259.388877:0:65914:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:2.0:1787826259.388881:0:65914:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd140d7fbb0 00010000:00010000:2.0:1787826259.388885:0:65914:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:2.0:1787826259.388892:0:65914:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:2.0:1787826259.388901:0:65914:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:2.0:1787826259.389033:0:65914:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:2.0:1787826259.389053:0:65914:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffc2de5e4ec 00000080:00200008:2.0:1787826259.389064:0:65914:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:3.0:1787826259.391194:0:65917:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826259.391227:0:65917:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826259.391243:0:65917:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00010000:00010000:3.0:1787826259.391258:0:65917:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:3.0:1787826259.391266:0:65917:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826259.391272:0:65917:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826259.391275:0:65917:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:3.0:1787826259.391282:0:65917:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141c7fbf8 00010000:00010000:3.0:1787826259.391286:0:65917:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00010000:00010000:3.0:1787826259.391294:0:65917:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:3.0:1787826259.391307:0:65917:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826259.391313:0:65917:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=27 00000080:00200000:3.0:1787826259.391318:0:65917:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 1 00000080:00200000:3.0:1787826259.391328:0:65917:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), file ffff907d83cc2780, open_flags 18800, mode 0 opened 0 00000080:00200000:3.0:1787826259.391334:0:65917:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), intent=open 00000002:00010000:3.0:1787826259.391346:0:65917:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204a.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1fc:0x0], intent: open flags 0304001 00010000:00010000:3.0:1787826259.391404:0:65917:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef400/0xe5f736940357b707 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 65917 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826259.391411:0:65917:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef400/0xe5f736940357b707 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65917 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826259.391422:0:65917:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826259.391432:0:65917:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83527c00 x1874670011401472/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826259.391442:0:65917:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef400/0xe5f736940357b707 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65917 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826259.391650:0:65917:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65917:1874670011401472:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826259.391825:0:65917:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d961476c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826259.404643:0:65917:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83527c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65917:1874670011401472:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826259.404668:0:65917:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef400/0xe5f736940357b707 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65917 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.404682:0:65917:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef400/0xe5f736940357b707 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 65917 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.404689:0:65917:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef400/0xe5f736940357b707 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 65917 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.404701:0:65917:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef400/0xe5f736940357b707 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 65917 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.404721:0:65917:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef400/0xe5f736940357b707 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 65917 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.404735:0:65917:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907da59ef400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826259.404739:0:65917:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef400/0xe5f736940357b707 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 65917 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.404746:0:65917:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef400/0xe5f736940357b707 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 65917 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.404753:0:65917:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef400/0xe5f736940357b707 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 65917 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.404760:0:65917:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826259.404767:0:65917:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef400/0xe5f736940357b707 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 65917 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.404773:0:65917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef400/0xe5f736940357b707 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 65917 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826259.404793:0:65917:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d83527c00 x1874670011401472/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826277 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826259.404806:0:65917:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1fc:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1787826259.405045:0:65917:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826259.405065:0:65917:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826259.405079:0:65917:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:0.0:1787826259.405093:0:65917:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:0.0:1787826259.405100:0:65917:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826259.405105:0:65917:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826259.405108:0:65917:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:0.0:1787826259.405113:0:65917:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141c7fbf8 00010000:00010000:0.0:1787826259.405117:0:65917:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:0.0:1787826259.405124:0:65917:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:0.0:1787826259.405138:0:65917:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826259.405146:0:65917:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=267 00000080:00200000:0.0:1787826259.405153:0:65917:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 44 00000080:00200008:0.0:1787826259.405161:0:65917:0:(file.c:1090:ll_file_open()) START file d204a.sanity-flr:[0x200000404:0x1fc:0x0](ffff907d845442c8), flags 304000 00000080:00200080:0.0:1787826259.405193:0:65917:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8) cmd=c1406916 arg=1712a20 00000100:00100000:0.0:1787826259.405249:0:65917:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65917:1874670011401600:192.168.203.104@tcp:34:lfs.0 00000100:00100000:0.0:1787826259.405338:0:65917:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d888670c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826259.417198:0:65917:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65917:1874670011401600:192.168.203.104@tcp:34:lfs.0 00000080:00200008:0.0:1787826259.417364:0:65917:0:(file.c:479:ll_file_release()) START file d204a.sanity-flr:[0x200000404:0x1fc:0x0](ffff907d845442c8), flags 304000 00010000:00010000:0.0:1787826259.417393:0:65917:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b6ac lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f77a expref: -99 pid: 65904 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200008:2.0:1787826259.458416:0:65921:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1787826259.458426:0:65921:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826259.458447:0:65921:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826259.458465:0:65921:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1787826259.458480:0:65921:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:2.0:1787826259.458488:0:65921:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826259.458496:0:65921:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826259.458499:0:65921:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:2.0:1787826259.458507:0:65921:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141cd3cb8 00010000:00010000:2.0:1787826259.458511:0:65921:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1787826259.458519:0:65921:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:2.0:1787826259.461847:0:65921:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826259.461875:0:65921:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826259.461892:0:65921:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1787826259.461908:0:65921:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:2.0:1787826259.461916:0:65921:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826259.461922:0:65921:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826259.461925:0:65921:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:2.0:1787826259.461932:0:65921:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141cd3bb0 00010000:00010000:2.0:1787826259.461935:0:65921:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1787826259.461944:0:65921:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:2.0:1787826259.462194:0:65921:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:2.0:1787826259.462205:0:65921:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:0.0:1787826259.465767:0:65921:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7fff899e3bec 00000080:00200008:0.0:1787826259.465794:0:65921:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:3.0:1787826259.471163:0:65926:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826259.471193:0:65926:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826259.471211:0:65926:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:3.0:1787826259.471227:0:65926:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:3.0:1787826259.471235:0:65926:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826259.471243:0:65926:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826259.471247:0:65926:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:3.0:1787826259.471254:0:65926:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141d13bf8 00010000:00010000:3.0:1787826259.471258:0:65926:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:3.0:1787826259.471267:0:65926:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:3.0:1787826259.471282:0:65926:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826259.471290:0:65926:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=27 00000080:00200000:3.0:1787826259.471295:0:65926:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 1 00000080:00200000:3.0:1787826259.471306:0:65926:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), file ffff907d83cc2d80, open_flags 18800, mode 0 opened 0 00000080:00200000:3.0:1787826259.471313:0:65926:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), intent=open 00000002:00010000:3.0:1787826259.471325:0:65926:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204a.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1fc:0x0], intent: open flags 0304001 00010000:00010000:3.0:1787826259.471374:0:65926:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef800/0xe5f736940357b70e lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 65926 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826259.471383:0:65926:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef800/0xe5f736940357b70e lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65926 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826259.471395:0:65926:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826259.471407:0:65926:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83525e00 x1874670011401856/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826259.471417:0:65926:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef800/0xe5f736940357b70e lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65926 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826259.471438:0:65926:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65926:1874670011401856:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826259.473089:0:65926:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d96147f00 going to sleep for 13 seconds 00000100:00100000:0.0:1787826259.480487:0:65926:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65926:1874670011401856:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826259.480525:0:65926:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef800/0xe5f736940357b70e lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65926 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.480540:0:65926:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef800/0xe5f736940357b70e lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 65926 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.480548:0:65926:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef800/0xe5f736940357b70e lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 65926 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.480555:0:65926:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef800/0xe5f736940357b70e lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 65926 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.480563:0:65926:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef800/0xe5f736940357b70e lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 65926 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.480579:0:65926:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907da59ef800 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826259.480585:0:65926:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef800/0xe5f736940357b70e lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 65926 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.480593:0:65926:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef800/0xe5f736940357b70e lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 65926 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.480600:0:65926:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef800/0xe5f736940357b70e lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 65926 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.480632:0:65926:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826259.480641:0:65926:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef800/0xe5f736940357b70e lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 65926 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.480648:0:65926:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef800/0xe5f736940357b70e lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 65926 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826259.480670:0:65926:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d83525e00 x1874670011401856/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826277 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826259.480684:0:65926:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1fc:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1787826259.480937:0:65926:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826259.480955:0:65926:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826259.480970:0:65926:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:0.0:1787826259.480983:0:65926:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:0.0:1787826259.480990:0:65926:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826259.480995:0:65926:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826259.480999:0:65926:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:0.0:1787826259.481003:0:65926:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141d13bf8 00010000:00010000:0.0:1787826259.481007:0:65926:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:0.0:1787826259.481015:0:65926:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:0.0:1787826259.481027:0:65926:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826259.481034:0:65926:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=267 00000080:00200000:0.0:1787826259.481041:0:65926:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 44 00000080:00200008:0.0:1787826259.481050:0:65926:0:(file.c:1090:ll_file_open()) START file d204a.sanity-flr:[0x200000404:0x1fc:0x0](ffff907d845442c8), flags 304000 00000080:00200080:0.0:1787826259.481085:0:65926:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8) cmd=c1406916 arg=15c2a20 00000100:00100000:0.0:1787826259.481138:0:65926:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65926:1874670011401984:192.168.203.104@tcp:34:lfs.0 00000100:00100000:0.0:1787826259.481226:0:65926:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d888670c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826259.489355:0:65926:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65926:1874670011401984:192.168.203.104@tcp:34:lfs.0 00000080:00200008:0.0:1787826259.489521:0:65926:0:(file.c:479:ll_file_release()) START file d204a.sanity-flr:[0x200000404:0x1fc:0x0](ffff907d845442c8), flags 304000 00010000:00010000:0.0:1787826259.489549:0:65926:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b6ac lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f77a expref: -99 pid: 65904 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200008:2.0:1787826259.501396:0:65928:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1787826259.501405:0:65928:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826259.501424:0:65928:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826259.501442:0:65928:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:2.0:1787826259.501456:0:65928:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:2.0:1787826259.501464:0:65928:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826259.501472:0:65928:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826259.501475:0:65928:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:2.0:1787826259.501483:0:65928:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141493cb8 00010000:00010000:2.0:1787826259.501487:0:65928:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:2.0:1787826259.501497:0:65928:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00000080:00200000:2.0:1787826259.501773:0:65928:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826259.501783:0:65928:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826259.501791:0:65928:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00010000:00010000:2.0:1787826259.501801:0:65928:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00000080:00200000:2.0:1787826259.501808:0:65928:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826259.501813:0:65928:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826259.501816:0:65928:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:2.0:1787826259.501820:0:65928:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141493bb0 00010000:00010000:2.0:1787826259.501824:0:65928:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00010000:00010000:2.0:1787826259.501832:0:65928:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000080:00200000:2.0:1787826259.501842:0:65928:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:2.0:1787826259.501848:0:65928:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:2.0:1787826259.501863:0:65928:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffffdc15cfc 00000080:00200008:2.0:1787826259.501874:0:65928:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:1.0:1787826259.502857:0:65932:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826259.502873:0:65932:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826259.502897:0:65932:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:1.0:1787826259.502910:0:65932:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000080:00200000:1.0:1787826259.502919:0:65932:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826259.502926:0:65932:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826259.502930:0:65932:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:1.0:1787826259.502936:0:65932:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141d2bbf8 00010000:00010000:1.0:1787826259.502940:0:65932:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:1.0:1787826259.502948:0:65932:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00000080:00200000:1.0:1787826259.502960:0:65932:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826259.502967:0:65932:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=27 00000080:00200000:1.0:1787826259.502972:0:65932:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 1 00000080:00200000:1.0:1787826259.502982:0:65932:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), file ffff907d91739680, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1787826259.502989:0:65932:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), intent=open 00000002:00010000:1.0:1787826259.503003:0:65932:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204a.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1fc:0x0], intent: open flags 0304001 00010000:00010000:1.0:1787826259.503053:0:65932:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b715 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 65932 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.503061:0:65932:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b715 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65932 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826259.503073:0:65932:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826259.503085:0:65932:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a2940 x1874670011402240/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826259.503095:0:65932:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b715 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65932 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826259.503117:0:65932:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65932:1874670011402240:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826259.503778:0:65932:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb600 going to sleep for 13 seconds 00000100:00100000:1.0:1787826259.510393:0:65932:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65932:1874670011402240:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826259.510417:0:65932:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b715 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65932 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.510430:0:65932:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b715 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 65932 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.510438:0:65932:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b715 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 65932 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.510450:0:65932:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b715 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 65932 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.510460:0:65932:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b715 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 65932 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.510467:0:65932:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0a400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826259.510472:0:65932:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b715 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 65932 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.510479:0:65932:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b715 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 65932 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.510487:0:65932:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b715 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 65932 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.510494:0:65932:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826259.510501:0:65932:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b715 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 65932 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826259.510508:0:65932:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b715 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 65932 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826259.510526:0:65932:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908a2940 x1874670011402240/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826277 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826259.510538:0:65932:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1fc:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826259.510828:0:65932:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826259.510846:0:65932:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826259.510860:0:65932:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00010000:00010000:1.0:1787826259.510873:0:65932:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00000080:00200000:1.0:1787826259.510881:0:65932:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826259.510886:0:65932:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826259.510890:0:65932:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:1.0:1787826259.510895:0:65932:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141d2bbf8 00010000:00010000:1.0:1787826259.510899:0:65932:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00010000:00010000:1.0:1787826259.510907:0:65932:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00000080:00200000:1.0:1787826259.510922:0:65932:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826259.510929:0:65932:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=267 00000080:00200000:1.0:1787826259.510935:0:65932:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 44 00000080:00200008:1.0:1787826259.510942:0:65932:0:(file.c:1090:ll_file_open()) START file d204a.sanity-flr:[0x200000404:0x1fc:0x0](ffff907d845442c8), flags 304000 00000080:00200080:1.0:1787826259.510973:0:65932:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8) cmd=c1406916 arg=1b23a20 00000100:00100000:1.0:1787826259.511029:0:65932:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65932:1874670011402368:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826259.511101:0:65932:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb600 going to sleep for 13 seconds 00000100:00100000:1.0:1787826259.525838:0:65932:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65932:1874670011402368:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826259.525899:0:65932:0:(file.c:479:ll_file_release()) START file d204a.sanity-flr:[0x200000404:0x1fc:0x0](ffff907d845442c8), flags 304000 00010000:00010000:1.0:1787826259.525926:0:65932:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b6ac lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f77a expref: -99 pid: 65904 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200008:2.0:1787826259.545439:0:65935:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1787826259.545448:0:65935:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826259.545466:0:65935:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826259.545484:0:65935:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:2.0:1787826259.545500:0:65935:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00000080:00200000:2.0:1787826259.545509:0:65935:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826259.545516:0:65935:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826259.545520:0:65935:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:2.0:1787826259.545527:0:65935:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141d13cb8 00010000:00010000:2.0:1787826259.545531:0:65935:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:2.0:1787826259.545541:0:65935:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00000080:00200000:2.0:1787826259.545850:0:65935:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826259.545858:0:65935:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826259.545866:0:65935:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00010000:00010000:2.0:1787826259.545875:0:65935:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00000080:00200000:2.0:1787826259.545883:0:65935:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826259.545888:0:65935:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826259.545890:0:65935:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:2.0:1787826259.545895:0:65935:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141d13bb0 00010000:00010000:2.0:1787826259.545898:0:65935:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00010000:00010000:2.0:1787826259.545906:0:65935:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00000080:00200000:2.0:1787826259.545915:0:65935:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:2.0:1787826259.545922:0:65935:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:2.0:1787826259.545936:0:65935:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7fff2c5ad5cc 00000080:00200008:2.0:1787826259.545948:0:65935:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:0.0:1787826259.549773:0:65940:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826259.549799:0:65940:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826259.549816:0:65940:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00010000:00010000:0.0:1787826259.549830:0:65940:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00000080:00200000:0.0:1787826259.549839:0:65940:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826259.549846:0:65940:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826259.549849:0:65940:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:0.0:1787826259.549855:0:65940:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141dc3bf8 00010000:00010000:0.0:1787826259.549859:0:65940:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00010000:00010000:0.0:1787826259.549868:0:65940:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00000080:00200000:0.0:1787826259.549886:0:65940:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826259.549892:0:65940:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=27 00000080:00200000:0.0:1787826259.549897:0:65940:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 1 00000080:00200000:0.0:1787826259.549909:0:65940:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), file ffff907d9245ec00, open_flags 18800, mode 0 opened 0 00000080:00200000:0.0:1787826259.549914:0:65940:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), intent=open 00000002:00010000:0.0:1787826259.549926:0:65940:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204a.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1fc:0x0], intent: open flags 0304001 00010000:00010000:0.0:1787826259.549974:0:65940:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b71c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 65940 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.549982:0:65940:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b71c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65940 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826259.549993:0:65940:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826259.550003:0:65940:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90847840 x1874670011402624/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826259.550012:0:65940:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b71c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65940 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826259.550033:0:65940:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65940:1874670011402624:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826259.550837:0:65940:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d888670c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826259.555643:0:65940:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65940:1874670011402624:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826259.555666:0:65940:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b71c lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65940 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.555679:0:65940:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b71c lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 65940 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.555692:0:65940:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b71c lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 65940 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.555700:0:65940:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b71c lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 65940 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.555709:0:65940:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b71c lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 65940 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.555716:0:65940:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a000 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826259.555721:0:65940:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b71c lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 65940 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.555728:0:65940:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b71c lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 65940 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.555735:0:65940:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b71c lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 65940 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.555742:0:65940:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826259.555750:0:65940:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b71c lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 65940 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.555757:0:65940:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b71c lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 65940 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826259.555774:0:65940:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d90847840 x1874670011402624/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826277 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826259.555787:0:65940:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1fc:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1787826259.556001:0:65940:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826259.556019:0:65940:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826259.556034:0:65940:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00010000:00010000:0.0:1787826259.556045:0:65940:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00000080:00200000:0.0:1787826259.556052:0:65940:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826259.556057:0:65940:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826259.556061:0:65940:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:0.0:1787826259.556065:0:65940:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141dc3bf8 00010000:00010000:0.0:1787826259.556068:0:65940:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00010000:00010000:0.0:1787826259.556075:0:65940:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00000080:00200000:0.0:1787826259.556088:0:65940:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826259.556095:0:65940:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=267 00000080:00200000:0.0:1787826259.556102:0:65940:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 44 00000080:00200008:0.0:1787826259.556110:0:65940:0:(file.c:1090:ll_file_open()) START file d204a.sanity-flr:[0x200000404:0x1fc:0x0](ffff907d845442c8), flags 304000 00000080:00200080:0.0:1787826259.556154:0:65940:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8) cmd=c1406916 arg=2634a20 00000100:00100000:0.0:1787826259.556210:0:65940:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65940:1874670011402752:192.168.203.104@tcp:34:lfs.0 00000100:00100000:0.0:1787826259.556288:0:65940:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d888670c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826259.559151:0:65940:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65940:1874670011402752:192.168.203.104@tcp:34:lfs.0 00000080:00200008:0.0:1787826259.559321:0:65940:0:(file.c:479:ll_file_release()) START file d204a.sanity-flr:[0x200000404:0x1fc:0x0](ffff907d845442c8), flags 304000 00010000:00010000:0.0:1787826259.559350:0:65940:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b6ac lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f77a expref: -99 pid: 65904 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:0.0:1787826259.574201:0:65941:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826259.574228:0:65941:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826259.574245:0:65941:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00010000:00010000:0.0:1787826259.574262:0:65941:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00000080:00200000:0.0:1787826259.574270:0:65941:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826259.574278:0:65941:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826259.574281:0:65941:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:0.0:1787826259.574288:0:65941:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141d13bf8 00010000:00010000:0.0:1787826259.574292:0:65941:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00010000:00010000:0.0:1787826259.574301:0:65941:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00000080:00200000:0.0:1787826259.574315:0:65941:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826259.574322:0:65941:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=27 00000080:00200000:0.0:1787826259.574327:0:65941:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 1 00000080:00200000:0.0:1787826259.574338:0:65941:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), file ffff907d85050d80, open_flags 18800, mode 0 opened 0 00000080:00200000:0.0:1787826259.574344:0:65941:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), intent=open 00000002:00010000:0.0:1787826259.574417:0:65941:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204a.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1fc:0x0], intent: open flags 0304001 00010000:00010000:0.0:1787826259.574466:0:65941:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b723 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 65941 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.574478:0:65941:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b723 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65941 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826259.574489:0:65941:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826259.574501:0:65941:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90847840 x1874670011403008/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826259.574510:0:65941:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b723 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65941 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826259.574530:0:65941:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65941:1874670011403008:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826259.575066:0:65941:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d888670c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826259.578811:0:65941:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65941:1874670011403008:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826259.578834:0:65941:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b723 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 65941 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.578848:0:65941:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b723 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 65941 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.578855:0:65941:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b723 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 65941 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.578862:0:65941:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b723 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 65941 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.578869:0:65941:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b723 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 65941 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.578875:0:65941:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a000 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826259.578881:0:65941:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b723 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 65941 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.578887:0:65941:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b723 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 65941 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.578894:0:65941:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b723 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 65941 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.578901:0:65941:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826259.578908:0:65941:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b723 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 65941 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826259.578914:0:65941:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b723 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 65941 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826259.578933:0:65941:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d90847840 x1874670011403008/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826277 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826259.578945:0:65941:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1fc:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1787826259.579066:0:65941:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826259.579084:0:65941:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826259.579097:0:65941:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00010000:00010000:0.0:1787826259.579109:0:65941:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00000080:00200000:0.0:1787826259.579116:0:65941:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826259.579123:0:65941:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826259.579126:0:65941:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:0.0:1787826259.579131:0:65941:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141d13bf8 00010000:00010000:0.0:1787826259.579134:0:65941:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00010000:00010000:0.0:1787826259.579155:0:65941:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00000080:00200000:0.0:1787826259.579169:0:65941:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826259.579177:0:65941:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=267 00000080:00200000:0.0:1787826259.579184:0:65941:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 44 00000080:00200008:0.0:1787826259.579193:0:65941:0:(file.c:1090:ll_file_open()) START file d204a.sanity-flr:[0x200000404:0x1fc:0x0](ffff907d845442c8), flags 304000 00000080:00200080:0.0:1787826259.579229:0:65941:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8) cmd=c0086915 arg=245b3d0 00000100:00100000:0.0:1787826259.579286:0:65941:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65941:1874670011403136:192.168.203.104@tcp:34:lfs.0 00000100:00100000:0.0:1787826259.579829:0:65941:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d888670c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826259.586394:0:65941:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:65941:1874670011403136:192.168.203.104@tcp:34:lfs.0 00000080:00200008:0.0:1787826259.586551:0:65941:0:(file.c:479:ll_file_release()) START file d204a.sanity-flr:[0x200000404:0x1fc:0x0](ffff907d845442c8), flags 304000 00010000:00010000:0.0:1787826259.586592:0:65941:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ac00/0xe5f736940357b6ac lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f77a expref: -99 pid: 65904 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1787826259.830105:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826260.852791:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826261.750044:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011403264:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826261.750163:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011403520:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826261.750202:0:2351:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1787826261.750208:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011403392:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826261.750234:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835243c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011403648:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826261.754221:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011403264:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826261.758728:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83527c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011403392:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826261.764788:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835243c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011403648:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826261.764809:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011403520:192.168.203.104@tcp:400:kworker.0 00010000:00010000:1.0:1787826261.877015:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826262.900759:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826263.931676:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826264.949871:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1787826265.069803:0:66359:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826265.069833:0:66359:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826265.069852:0:66359:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00010000:00010000:2.0:1787826265.069874:0:66359:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00000080:00200000:2.0:1787826265.069883:0:66359:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826265.069890:0:66359:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826265.069893:0:66359:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:2.0:1787826265.069901:0:66359:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141e27b58 00010000:00010000:2.0:1787826265.069905:0:66359:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00010000:00010000:2.0:1787826265.069914:0:66359:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00000080:00200000:2.0:1787826265.069931:0:66359:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826265.069938:0:66359:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=24 00000080:00200000:2.0:1787826265.069943:0:66359:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 1 00000080:00200000:2.0:1787826265.069954:0:66359:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), flags=8 00000080:00200000:2.0:1787826265.069958:0:66359:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8), intent=getattr 00000002:00010000:2.0:1787826265.069970:0:66359:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204a.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1fc:0x0], intent: getattr flags 01 00010000:00010000:2.0:1787826265.070016:0:66359:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 66359 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826265.070025:0:66359:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66359 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826265.070035:0:66359:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66359 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826265.070064:0:66359:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149e00 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:66359:1874670011403776:192.168.203.104@tcp:101:rm.0 00000100:00100000:2.0:1787826265.070211:0:66359:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88963480 going to sleep for 13 seconds 00000100:00100000:2.0:1787826265.072480:0:66359:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149e00 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:66359:1874670011403776:192.168.203.104@tcp:101:rm.0 00010000:00010000:2.0:1787826265.072505:0:66359:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1fc:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f96b expref: -99 pid: 66359 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826265.072543:0:66359:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f96b expref: -99 pid: 66359 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826265.072552:0:66359:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f96b expref: -99 pid: 66359 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826265.072565:0:66359:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f96b expref: -99 pid: 66359 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826265.072575:0:66359:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f96b expref: -99 pid: 66359 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826265.072586:0:66359:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f96b expref: -99 pid: 66359 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826265.072594:0:66359:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f96b expref: -99 pid: 66359 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826265.072601:0:66359:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f96b expref: -99 pid: 66359 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826265.072857:0:66359:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d91149e00 x1874670011403776/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/1136 e 0 to 0 dl 1787826283 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1787826265.072868:0:66359:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f96b expref: -99 pid: 66359 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1787826265.072882:0:66359:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f96b expref: -99 pid: 66359 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826265.072893:0:66359:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/509 (0 0) 00000080:00200000:2.0:1787826265.072900:0:66359:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1fc:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1787826265.072954:0:66359:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x1fd:0x0], updating i_size 2097152 00000080:00200000:2.0:1787826265.072958:0:66359:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x1fd:0x0](ffff907d92219148): rc = 0 00000080:00200000:2.0:1787826265.072975:0:66359:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x1fd:0x0]: layout version change: 8 -> 8 00000080:00010000:2.0:1787826265.072980:0:66359:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) for lock 0xe5f736940357b72a 00000080:00010000:2.0:1787826265.072991:0:66359:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) 00000080:00010000:2.0:1787826265.072994:0:66359:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b72a from it ffff9dd141e27b20 00010000:00010000:2.0:1787826265.072998:0:66359:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f96b expref: -99 pid: 66359 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826265.073007:0:66359:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f96b expref: -99 pid: 66359 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1787826265.073041:0:66359:0:(file.c:6313:ll_getattr_dentry()) START file f204a.sanity-flr:[0x200000404:0x1fd:0x0](ffff907d92219148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1787826265.073048:0:66359:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148),name=f204a.sanity-flr 00000002:00010000:2.0:1787826265.073065:0:66359:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1fd:0x0]) in obj [0x200000404:0x1fc:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826265.073074:0:66359:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f96b expref: -99 pid: 66359 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826265.073085:0:66359:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f96b expref: -99 pid: 66359 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826265.073092:0:66359:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1fc:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826265.073096:0:66359:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) 00000080:00010000:2.0:1787826265.073099:0:66359:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1fd:0x0](ffff907d92219148) for lock 0xe5f736940357b72a 00000080:00010000:2.0:1787826265.073103:0:66359:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b72a from it ffff9dd141e27cb8 00010000:00010000:2.0:1787826265.073106:0:66359:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0f96b expref: -99 pid: 66359 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826265.073114:0:66359:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0f96b expref: -99 pid: 66359 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1787826265.073166:0:66359:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826265.073174:0:66359:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826265.073182:0:66359:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00010000:00010000:2.0:1787826265.073190:0:66359:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00000080:00200000:2.0:1787826265.073196:0:66359:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826265.073200:0:66359:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826265.073202:0:66359:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:2.0:1787826265.073206:0:66359:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141e27c10 00010000:00010000:2.0:1787826265.073209:0:66359:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00010000:00010000:2.0:1787826265.073217:0:66359:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00000080:00200000:2.0:1787826265.073226:0:66359:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826265.073232:0:66359:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=24 00000080:00200000:2.0:1787826265.073236:0:66359:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 1 00000080:00200000:2.0:1787826265.073242:0:66359:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204a.sanity-flr, flags=0 00000080:00200000:2.0:1787826265.073246:0:66359:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1fc:0x0](ffff907d845442c8), inode mode 41ed mask 3 00000080:00200000:2.0:1787826265.073250:0:66359:0:(namei.c:2418:ll_unlink()) VFS Op:name=f204a.sanity-flr, dir=[0x200000404:0x1fc:0x0](ffff907d845442c8) 00010000:00010000:2.0:1787826265.073274:0:66359:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0f87d expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1787826265.073286:0:66359:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x1fd:0x0]: losing layout lock 00010000:00010000:2.0:1787826265.073296:0:66359:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0f96b expref: -99 pid: 66359 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1787826265.073303:0:66359:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x1fd:0x0]: losing layout lock 00010000:00010000:2.0:1787826265.073310:0:66359:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b690 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x20/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc0f718 expref: -99 pid: 65896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826265.073361:0:66359:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 1/0,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0f87d expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826265.073368:0:66359:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 1/0,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0f96b expref: -99 pid: 66359 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826265.073373:0:66359:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b690 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0f718 expref: -99 pid: 65896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826265.073379:0:66359:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1787826265.073381:0:66359:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b700 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1fd:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0f87d expref: -99 pid: 65912 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826265.073396:0:66359:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639be00/0xe5f736940357b72a lrc: 0/0,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0f96b expref: -99 pid: 66359 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826265.073406:0:66359:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b800/0xe5f736940357b690 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x1fd:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0f718 expref: -99 pid: 65896 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000020:00100000:2.0:1787826265.073424:0:66359:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1787826265.073432:0:66359:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d91149e00 x1874670011404032/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 512/2200 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787826265.073453:0:66359:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149e00 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:66359:1874670011404032:192.168.203.104@tcp:36:rm.0 00000100:00100000:2.0:1787826265.073592:0:66359:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88963480 going to sleep for 13 seconds 00000100:00100000:0.0:1787826265.077118:0:66359:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149e00 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:66359:1874670011404032:192.168.203.104@tcp:36:rm.0 00000080:00200000:0.0:1787826265.077176:0:66359:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fd:0x0] ignore/verify layout 1/0, layout version 7 restore needed 0 00020000:00200000:0.0:1787826265.077187:0:66359:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x1fd:0x0]: flr state: 1, move mirror from 1 to 1, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826265.077191:0:66359:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826265.077197:0:66359:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x10 00020000:00200000:0.0:1787826265.077206:0:66359:0:(lov_io.c:1253:lov_io_iter_init()) component[3] flags 0x10 00020000:00200000:0.0:1787826265.077212:0:66359:0:(lov_io.c:1253:lov_io_iter_init()) component[4] flags 0x40 00000080:00200000:0.0:1787826265.077234:0:66359:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fd:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1787826265.077241:0:66359:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200000404:0x1fd:0x0](ffff907d92219148) 00000100:00100000:3.0:1787826265.080926:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826265.081110:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025483520 00000100:00100000:2.0:1787826265.081120:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025483520 00000100:00100000:2.0:1787826265.081171:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7130 00000100:00100000:2.0:1787826265.081182:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83526580 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:36729:x1874670025483520:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826265.081202:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x420000030000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826265.081282:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826265.081289:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83526580 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:36729:x1874670025483520:12345-192.168.203.104@tcp:104: Request processed in 110us (373us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826265.081301:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7130 00010000:00010000:0.0:1787826265.082067:0:61297:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826265.082076:0:61297:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x420000030000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826265.082087:0:61297:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883b000 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826265.082093:0:61297:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x428400030000 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826265.082141:0:61297:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826265.082149:0:61297:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826265.082172:0:61297:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826265.082181:0:61297:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b6f9 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x323:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc0f86f expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826265.082201:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826265.082203:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826265.082205:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826265.082213:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1787826265.082230:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011404160:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1787826265.082677:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826265.083158:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025483648 00000100:00100000:3.0:1787826265.083165:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025483648 00000100:00100000:3.0:1787826265.083210:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7131 00000100:00100000:3.0:1787826265.083219:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91148000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:28813:x1874670025483648:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826265.083232:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b6eb lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x307:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc0f85a expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826265.083291:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826265.083298:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91148000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:28813:x1874670025483648:12345-192.168.203.104@tcp:104: Request processed in 80us (623us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826265.083305:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7131 00010000:00010000:1.0:1787826265.083463:0:47852:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826265.083470:0:47852:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b6eb lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x307:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc0f85a expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826265.083478:0:47852:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883b200 already unused, calling callback (ffffffffc118e410) 00010000:00010000:1.0:1787826265.083483:0:47852:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b6eb lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x307:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x428400010000 nid: local remote: 0x7918213f4fc0f85a expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826265.083593:0:47852:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b6eb lrc: 2/0,0 mode: --/PW res: [0x280000400:0x307:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc0f85a expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826265.083602:0:47852:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826265.083648:0:47852:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b6eb lrc: 1/0,0 mode: --/PW res: [0x280000400:0x307:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc0f85a expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826265.083658:0:47852:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b6eb lrc: 0/0,0 mode: --/PW res: [0x280000400:0x307:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc0f85a expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826265.083676:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826265.083678:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826265.083680:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826265.083687:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1787826265.083699:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011404288:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826265.089034:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011404160:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826265.090985:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011404288:192.168.203.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1787826265.973046:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1787826266.611438:0:66498:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-flr test 204b: FLR write/stale/resync tests with self-extending primary ========================================================== 06:24:25 (1787826265) 00000100:00100000:1.0:1787826266.872542:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2348:1874670011404416:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826266.877629:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1787826266.877653:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011404544:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826266.878205:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011404800:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826266.878257:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1787826266.878264:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011404672:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826266.887708:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2348:1874670011404416:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826266.888730:0:2349:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff907d986d0000: committing for last_committed 21474840486 gen 3 00000100:00100000:0.0:1787826266.888751:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011404672:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826266.888764:0:2349:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0001-osc-ffff907d986d0000: committing for last_committed 17179873822 gen 1 00000100:00100000:0.0:1787826266.888772:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011404800:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826266.888911:0:2348:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589942952 gen 1 00000100:00100000:1.0:1787826266.888923:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011404544:192.168.203.104@tcp:400:kworker.0 00010000:00010000:0.0:1787826266.997020:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1787826267.022962:0:66517:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826267.022990:0:66517:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826267.023009:0:66517:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00010000:00010000:1.0:1787826267.023027:0:66517:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00000080:00200000:1.0:1787826267.023037:0:66517:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826267.023044:0:66517:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826267.023048:0:66517:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:1.0:1787826267.023056:0:66517:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141853be0 00010000:00010000:1.0:1787826267.023061:0:66517:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00010000:00010000:1.0:1787826267.023071:0:66517:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00000080:00200000:1.0:1787826267.023090:0:66517:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826267.023108:0:66517:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d204b.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), flags=1536 00000080:00200000:1.0:1787826267.023111:0:66517:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826267.023117:0:66517:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826267.023124:0:66517:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00010000:00010000:1.0:1787826267.023133:0:66517:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00000080:00200000:1.0:1787826267.023139:0:66517:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826267.023143:0:66517:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826267.023146:0:66517:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:1.0:1787826267.023149:0:66517:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141853ce0 00010000:00010000:1.0:1787826267.023152:0:66517:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00010000:00010000:1.0:1787826267.023159:0:66517:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00000080:00200000:1.0:1787826267.023167:0:66517:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 3 00000080:00200000:1.0:1787826267.023173:0:66517:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826267.023179:0:66517:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826267.023185:0:66517:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00010000:00010000:1.0:1787826267.023193:0:66517:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00000080:00200000:1.0:1787826267.023204:0:66517:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826267.023209:0:66517:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826267.023212:0:66517:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b64a 00000080:00010000:1.0:1787826267.023215:0:66517:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b64a from it ffff9dd141853dc8 00010000:00010000:1.0:1787826267.023218:0:66517:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00010000:00010000:1.0:1787826267.023226:0:66517:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00000080:00200000:1.0:1787826267.023234:0:66517:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 3 00000080:00200000:1.0:1787826267.023239:0:66517:0:(namei.c:2239:do_mkdir()) VFS Op:name=d204b.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148) 00010000:00010000:1.0:1787826267.023269:0:66517:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00010000:00010000:1.0:1787826267.024407:0:66517:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00010000:00010000:1.0:1787826267.024420:0:66517:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826267.024423:0:66517:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da62eb200/0xe5f736940357b64a lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0f638 expref: -99 pid: 65895 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00000020:00100000:1.0:1787826267.024452:0:66517:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1787826267.024465:0:66517:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a03c0 x1874670011405056/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 568/2152 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826267.024491:0:66517:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job mkdir:6f891bcb-15cd-41e0-96e2-db843619c1be:66517:1874670011405056:192.168.203.104@tcp:36:mkdir.0 00000100:00100000:1.0:1787826267.025572:0:66517:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe80c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826267.032893:0:66517:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job mkdir:6f891bcb-15cd-41e0-96e2-db843619c1be:66517:1874670011405056:192.168.203.104@tcp:36:mkdir.0 00000080:00200000:1.0:1787826267.032937:0:66517:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8) 00000080:00200000:1.0:1787826267.032942:0:66517:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x1ff:0x0], updating i_size 4096 00000080:00200000:2.0:1787826267.079957:0:66520:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826267.079986:0:66520:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826267.080045:0:66520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.080056:0:66520:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.080065:0:66520:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826267.080089:0:66520:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66520:1874670011405184:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826267.080187:0:66520:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb3c0 going to sleep for 13 seconds 00000100:00100000:2.0:1787826267.084375:0:66520:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66520:1874670011405184:192.168.203.104@tcp:101:lfs.0 00010000:00010000:2.0:1787826267.084398:0:66520:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.084410:0:66520:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.084422:0:66520:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.084433:0:66520:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.084450:0:66520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.084458:0:66520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.084466:0:66520:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826267.084476:0:66520:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d9114bc00 x1874670011405184/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/688 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1787826267.084490:0:66520:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.084502:0:66520:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1787826267.084508:0:66520:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1787826267.084545:0:66520:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:2.0:1787826267.084551:0:66520:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:2.0:1787826267.084556:0:66520:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826267.084559:0:66520:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141f9bb58 00010000:00010000:2.0:1787826267.084563:0:66520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.084571:0:66520:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826267.084596:0:66520:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826267.084830:0:66520:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d204b.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), flags=29 00000080:00200000:3.0:1787826267.084836:0:66520:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d204b.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=0 00000002:00010000:3.0:1787826267.084852:0:66520:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: d204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00010000:00010000:3.0:1787826267.084887:0:66520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59efe00/0xe5f736940357b738 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826267.084895:0:66520:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59efe00/0xe5f736940357b738 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826267.084903:0:66520:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59efe00/0xe5f736940357b738 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826267.084921:0:66520:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66520:1874670011405312:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826267.084989:0:66520:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7d3c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826267.112268:0:66520:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66520:1874670011405312:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826267.112296:0:66520:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59efe00/0xe5f736940357b738 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9cd expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.112321:0:66520:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59efe00/0xe5f736940357b738 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9cd expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.112331:0:66520:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59efe00/0xe5f736940357b738 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9cd expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.112343:0:66520:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59efe00/0xe5f736940357b738 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9cd expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.112354:0:66520:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59efe00/0xe5f736940357b738 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9cd expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.112367:0:66520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59efe00/0xe5f736940357b738 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9cd expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.112378:0:66520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59efe00/0xe5f736940357b738 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9cd expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.112385:0:66520:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59efe00/0xe5f736940357b738 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9cd expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826267.112397:0:66520:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x10 disposition=0xb, status=0 req@ffff907d83525a40 x1874670011405312/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/584 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826267.112408:0:66520:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59efe00/0xe5f736940357b738 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9cd expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.112419:0:66520:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/511 (0 0) 00000080:00200000:0.0:1787826267.112426:0:66520:0:(namei.c:952:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1787826267.112447:0:66520:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x1ff:0x0], updating i_size 4096 00000080:00200000:0.0:1787826267.112451:0:66520:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x1ff:0x0](ffff907d9221e3c8): rc = 0 00000080:00010000:0.0:1787826267.112456:0:66520:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1ff:0x0](ffff907d9221e3c8) for lock 0xe5f736940357b738 00000080:00010000:0.0:1787826267.112468:0:66520:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1ff:0x0](ffff907d9221e3c8) 00000080:00010000:0.0:1787826267.112472:0:66520:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b738 from it ffff9dd141f9ba28 00010000:00010000:0.0:1787826267.112475:0:66520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59efe00/0xe5f736940357b738 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9cd expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.112483:0:66520:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59efe00/0xe5f736940357b738 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9cd expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826267.112528:0:66520:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:0.0:1787826267.112547:0:66520:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags=13 00000080:00200000:0.0:1787826267.112551:0:66520:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=getattr 00000002:00010000:0.0:1787826267.112568:0:66520:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1ff:0x0], intent: getattr flags 01 00010000:00010000:0.0:1787826267.112638:0:66520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b73f lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.112646:0:66520:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b73f lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.112655:0:66520:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b73f lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826267.112676:0:66520:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66520:1874670011405440:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826267.112783:0:66520:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd0c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826267.135673:0:66520:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66520:1874670011405440:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826267.135696:0:66520:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b73f lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.135711:0:66520:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b73f lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.135718:0:66520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b73f lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.135724:0:66520:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b73f lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.135752:0:66520:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b73f lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.135759:0:66520:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883b400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826267.135763:0:66520:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b73f lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.135769:0:66520:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b73f lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.135776:0:66520:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b73f lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.135804:0:66520:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826267.135813:0:66520:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b73f lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.135825:0:66520:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b73f lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826267.135868:0:66520:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff907d90844b40 x1874670011405440/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/584 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826267.135901:0:66520:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1ff:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1787826267.135936:0:66520:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 30 res 8589935620/511 (0 0) 00000080:00200000:0.0:1787826267.136017:0:66520:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826267.136034:0:66520:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826267.136046:0:66520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.136062:0:66520:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826267.136069:0:66520:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826267.136075:0:66520:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826267.136079:0:66520:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826267.136087:0:66520:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141f9bbe8 00010000:00010000:0.0:1787826267.136091:0:66520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.136103:0:66520:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826267.136126:0:66520:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826267.136133:0:66520:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=16408 00000080:00200000:0.0:1787826267.136138:0:66520:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:0.0:1787826267.136151:0:66520:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags=16392 00000080:00200000:0.0:1787826267.136156:0:66520:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=getattr 00000002:00010000:0.0:1787826267.136178:0:66520:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1ff:0x0], intent: getattr flags 01 00010000:00010000:0.0:1787826267.136221:0:66520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b746 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.136230:0:66520:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b746 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.136240:0:66520:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b746 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826267.136269:0:66520:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66520:1874670011405568:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826267.136358:0:66520:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd0c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826267.151341:0:66520:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66520:1874670011405568:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826267.151365:0:66520:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b746 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.151378:0:66520:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b746 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.151385:0:66520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b746 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.151392:0:66520:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b746 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.151400:0:66520:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b746 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.151407:0:66520:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883ae00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826267.151412:0:66520:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b746 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.151419:0:66520:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b746 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.151426:0:66520:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b746 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.151434:0:66520:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826267.151441:0:66520:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b746 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.151448:0:66520:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b746 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826267.151466:0:66520:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff907d90844b40 x1874670011405568/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/584 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826267.151480:0:66520:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1ff:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1787826267.151502:0:66520:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 30 res 8589935620/511 (0 0) 00000080:00200000:0.0:1787826267.151584:0:66520:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826267.151599:0:66520:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826267.151658:0:66520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.151670:0:66520:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826267.151677:0:66520:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826267.151682:0:66520:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826267.151685:0:66520:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826267.151691:0:66520:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141f9bb58 00010000:00010000:0.0:1787826267.151694:0:66520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.151702:0:66520:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1787826267.151716:0:66520:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826267.151725:0:66520:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=13 00000080:00200008:0.0:1787826267.151742:0:66520:0:(file.c:6313:ll_getattr_dentry()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1787826267.151748:0:66520:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8),name=d204b.sanity-flr 00000002:00010000:0.0:1787826267.151759:0:66520:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1ff:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826267.151765:0:66520:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 30 res 8589935620/511 (0 0) 00010000:00010000:0.0:1787826267.151808:0:66520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.151814:0:66520:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.151822:0:66520:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826267.151841:0:66520:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66520:1874670011405696:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826267.151923:0:66520:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd0c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826267.155260:0:66520:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66520:1874670011405696:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826267.155297:0:66520:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa05 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.155318:0:66520:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa05 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.155327:0:66520:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 4/1,0 mode: --/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa05 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.155340:0:66520:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa05 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.155359:0:66520:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa05 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.155372:0:66520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa05 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.155382:0:66520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa05 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.155391:0:66520:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa05 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826267.155405:0:66520:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d90844b40 x1874670011405696/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/584 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826267.155419:0:66520:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa05 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.155430:0:66520:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/511 (0 0) 00000080:00200000:0.0:1787826267.155438:0:66520:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1787826267.155467:0:66520:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x1ff:0x0], updating i_size 4096 00000080:00010000:0.0:1787826267.155474:0:66520:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1ff:0x0](ffff907d9221e3c8) for lock 0xe5f736940357b74d 00000080:00010000:0.0:1787826267.155483:0:66520:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x1ff:0x0](ffff907d9221e3c8) 00000080:00010000:0.0:1787826267.155487:0:66520:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b74d from it ffff9dd141f9bcb8 00010000:00010000:0.0:1787826267.155490:0:66520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa05 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.155505:0:66520:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa05 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:0.0:1787826267.156489:0:66520:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1787826267.156497:0:66520:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826267.156530:0:66520:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826267.156546:0:66520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826267.156558:0:66520:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1787826267.156565:0:66520:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826267.156571:0:66520:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826267.156575:0:66520:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826267.156581:0:66520:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141f9bcb8 00010000:00010000:0.0:1787826267.156584:0:66520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826267.156592:0:66520:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1787826267.158770:0:66520:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826267.158801:0:66520:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826267.158820:0:66520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1787826267.158843:0:66520:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1787826267.158851:0:66520:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826267.158860:0:66520:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826267.158864:0:66520:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826267.158872:0:66520:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141f9bb58 00010000:00010000:0.0:1787826267.158877:0:66520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1787826267.158887:0:66520:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:0.0:1787826267.158902:0:66520:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826267.158910:0:66520:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=24 00000080:00200000:0.0:1787826267.158917:0:66520:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:0.0:1787826267.158931:0:66520:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags=8 00000080:00200000:0.0:1787826267.158935:0:66520:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=getattr 00000002:00010000:0.0:1787826267.158949:0:66520:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1ff:0x0], intent: getattr flags 01 00010000:00010000:0.0:1787826267.159010:0:66520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b754 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.159019:0:66520:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b754 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.159032:0:66520:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b754 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826267.159071:0:66520:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66520:1874670011405824:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826267.159194:0:66520:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd0c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826267.164167:0:66520:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66520:1874670011405824:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826267.164198:0:66520:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b754 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.164214:0:66520:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b754 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.164222:0:66520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b754 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.164229:0:66520:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b754 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.164238:0:66520:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b754 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.164245:0:66520:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883b000 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826267.164251:0:66520:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b754 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.164258:0:66520:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b754 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.164265:0:66520:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b754 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x20e400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.164272:0:66520:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826267.164282:0:66520:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b754 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.164289:0:66520:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b754 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826267.164311:0:66520:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff907d90844b40 x1874670011405824/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/584 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826267.164327:0:66520:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1ff:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1787826267.164359:0:66520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa05 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.164374:0:66520:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa05 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00010000:0.0:1787826267.164384:0:66520:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b74d from it ffff9dd141f9b930 00010000:00010000:0.0:1787826267.164388:0:66520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa05 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.164396:0:66520:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa05 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826267.164503:0:66520:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826267.164535:0:66520:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826267.164544:0:66520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1787826267.164554:0:66520:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:0.0:1787826267.164565:0:66520:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826267.164571:0:66520:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826267.164574:0:66520:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826267.164580:0:66520:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141f9bbf8 00010000:00010000:0.0:1787826267.164583:0:66520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1787826267.164591:0:66520:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:0.0:1787826267.164600:0:66520:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826267.164991:0:66520:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=25 00000080:00200000:0.0:1787826267.165000:0:66520:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:0.0:1787826267.165009:0:66520:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204b.sanity-flr, flags=777 00000080:00200000:0.0:1787826267.165013:0:66520:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 3 00000080:00200000:0.0:1787826267.165018:0:66520:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), file ffff907d83962c00, open_flags a141, mode 81b6 opened 0 00000080:00200000:0.0:1787826267.165031:0:66520:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=open|creat 00000002:00010000:0.0:1787826267.165058:0:66520:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204b.sanity-flr,[0x200000404:0x200:0x0]) in obj [0x200000404:0x1ff:0x0], intent: open|creat flags 0120502 00010000:00010000:0.0:1787826267.165074:0:66520:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc0fa05 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.165093:0:66520:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59efe00/0xe5f736940357b738 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9cd expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.165142:0:66520:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 1/0,0 mode: --/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0fa05 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.165156:0:66520:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826267.165159:0:66520:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b74d lrc: 0/0,0 mode: --/PR res: [0x200000404:0x1ff:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0fa05 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.165188:0:66520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 3/0,1 mode: --/CW res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.165195:0:66520:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 3/0,1 mode: --/CW res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826267.165207:0:66520:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826267.165216:0:66520:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90844b40 x1874670011406080/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/67896 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826267.165225:0:66520:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 3/0,1 mode: --/CW res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826267.165249:0:66520:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66520:1874670011406080:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826267.165443:0:66520:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd0c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826267.169358:0:66520:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66520:1874670011406080:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826267.169397:0:66520:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 4/0,1 mode: --/CW res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa36 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.169428:0:66520:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 4/0,1 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa36 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.169437:0:66520:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 4/0,1 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa36 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.169454:0:66520:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa36 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.169466:0:66520:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa36 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.169480:0:66520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa36 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.169489:0:66520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa36 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.169496:0:66520:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa36 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826267.169509:0:66520:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x8000037, status=0 req@ffff907d90844b40 x1874670011406080/t8589942954(8589942954) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1787826267.169532:0:66520:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90844b40 x1874670011406080/t8589942954(8589942954) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826267.169546:0:66520:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa36 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.169558:0:66520:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/512 (0 0) 00000080:00200000:0.0:1787826267.169565:0:66520:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x200000404:0x1ff:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1787826267.169579:0:66520:0:(namei.c:1628:ll_create_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=open|creat 00000080:00200000:0.0:1787826267.169647:0:66520:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248) 00000080:00200000:0.0:1787826267.169656:0:66520:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x200:0x0], updating i_size 0 00000080:00200000:0.0:1787826267.169700:0:66520:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x200:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:0.0:1787826267.169705:0:66520:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x200000404:0x200:0x0](ffff907d93313248) 00000080:00010000:0.0:1787826267.169708:0:66520:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) for lock 0xe5f736940357b75b 00000080:00010000:0.0:1787826267.170368:0:66520:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) 00000080:00200008:0.0:1787826267.170388:0:66520:0:(file.c:1090:ll_file_open()) START file f204b.sanity-flr:[0x200000404:0x200:0x0](ffff907d93313248), flags 120501 00000002:00100000:2.0:1787826267.170661:0:66520:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90844b40 x1874670011406080/t8589942954(8589942954) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1787826267.170707:0:66520:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b75b from it ffff907d83241960 00010000:00010000:2.0:1787826267.170713:0:66520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fa36 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.170852:0:66520:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fa36 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826267.170881:0:66520:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248), inode mode 81a4 mask 40 00000080:00200000:2.0:1787826267.170956:0:66520:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248), inode mode 81a4 mask 2 00000080:00200000:2.0:1787826267.170968:0:66520:0:(xattr.c:414:ll_xattr_set()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248), xattr lov 00000002:00010000:2.0:1787826267.170991:0:66520:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x200:0x0]) in obj [0x200000404:0x1ff:0x0], intent: open flags 040000002 00010000:00010000:2.0:1787826267.171058:0:66520:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639ba00/0xe5f736940357b762 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.171066:0:66520:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639ba00/0xe5f736940357b762 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826267.171079:0:66520:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826267.171087:0:66520:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114bc00 x1874670011406336/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826267.171098:0:66520:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639ba00/0xe5f736940357b762 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826267.171124:0:66520:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66520:1874670011406336:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826267.171362:0:66520:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb3c0 going to sleep for 13 seconds 00000100:00100000:2.0:1787826267.175234:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826267.175331:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025484544 00000100:00100000:3.0:1787826267.175347:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025484544 00000100:00100000:3.0:1787826267.175457:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7132 00000100:00100000:3.0:1787826267.175469:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d9114a940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32456:x1874670025484544:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826267.175489:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0fa36 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826267.175597:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826267.175605:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d9114a940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32456:x1874670025484544:12345-192.168.203.104@tcp:104: Request processed in 139us (387us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826267.175644:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7132 00010000:00010000:1.0:1787826267.175710:0:60835:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826267.175717:0:60835:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0fa36 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826267.175726:0:60835:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883ae00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826267.175730:0:60835:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc0fa36 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826267.175739:0:60835:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc0fa36 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826267.175747:0:60835:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 3/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x7918213f4fc0fa36 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826267.175770:0:60835:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x200:0x0]: losing layout lock 00010000:00010000:1.0:1787826267.175822:0:60835:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 2/0,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0fa36 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826267.175829:0:60835:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826267.175862:0:60835:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 1/0,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0fa36 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826267.175871:0:60835:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b75b lrc: 0/0,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0fa36 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826267.175902:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826267.175905:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826267.175907:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826267.175918:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011406464:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826267.194429:0:66520:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66520:1874670011406336:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826267.194461:0:66520:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639ba00/0xe5f736940357b762 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.194476:0:66520:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639ba00/0xe5f736940357b762 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.194484:0:66520:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639ba00/0xe5f736940357b762 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.194492:0:66520:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639ba00/0xe5f736940357b762 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.194501:0:66520:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639ba00/0xe5f736940357b762 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.194507:0:66520:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907da639ba00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826267.194521:0:66520:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639ba00/0xe5f736940357b762 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.194528:0:66520:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639ba00/0xe5f736940357b762 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.194537:0:66520:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639ba00/0xe5f736940357b762 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.194545:0:66520:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826267.194553:0:66520:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639ba00/0xe5f736940357b762 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.194559:0:66520:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639ba00/0xe5f736940357b762 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 66520 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826267.194584:0:66520:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x800002b, status=0 req@ffff907d9114bc00 x1874670011406336/t8589942955(8589942955) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1160 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1787826267.194599:0:66520:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d9114bc00 x1874670011406336/t8589942955(8589942955) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1160 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826267.194677:0:66520:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1ff:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1787826267.194744:0:66520:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1787826267.194754:0:66520:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1787826267.194806:0:66520:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x200:0x0], updating i_size 0 00000002:00100000:0.0:1787826267.194823:0:66520:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d9114bc00 x1874670011406336/t8589942955(8589942955) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1787826267.194876:0:66520:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d9114bc00 x1874670011406336/t8589942955(8589942955) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1787826267.194876:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011406464:192.168.203.104@tcp:103:ldlm_bl.0 00000020:00100000:0.0:1787826267.194907:0:66520:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1787826267.194916:0:66520:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90844780 x1874670011406720/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1787826267.194942:0:66520:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66520:1874670011406720:192.168.203.104@tcp:35:lfs.0 00000100:00100000:0.0:1787826267.195032:0:66520:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd0c0 going to sleep for 11 seconds 00000100:00100000:0.0:1787826267.200534:0:66520:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66520:1874670011406720:192.168.203.104@tcp:35:lfs.0 00000002:00100000:0.0:1787826267.200571:0:66520:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d9114bc00 x1874670011406336/t8589942955(8589942955) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826285 ref 2 fl Complete:RQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1787826267.200935:0:66520:0:(file.c:479:ll_file_release()) START file f204b.sanity-flr:[0x200000404:0x200:0x0](ffff907d93313248), flags 100001 00000002:00100000:1.0:1787826267.201242:0:66520:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d90844b40 x1874670011406080/t8589942954(8589942954) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826285 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826267.201255:0:66520:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826267.201261:0:66520:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a1a40 x1874670011406976/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826267.201284:0:66520:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66520:1874670011406976:192.168.203.104@tcp:35:lfs.0 00000100:00100000:1.0:1787826267.201457:0:66520:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe80c0 going to sleep for 11 seconds 00000100:00100000:1.0:1787826267.214262:0:66520:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66520:1874670011406976:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826267.214297:0:66520:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d90844b40 x1874670011406080/t8589942954(8589942954) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826267.225063:0:66521:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826267.225093:0:66521:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826267.225113:0:66521:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826267.225131:0:66521:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1787826267.225140:0:66521:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826267.225149:0:66521:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826267.225153:0:66521:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:1.0:1787826267.225171:0:66521:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd14186fbf8 00010000:00010000:1.0:1787826267.225175:0:66521:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826267.225185:0:66521:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1787826267.225200:0:66521:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826267.225501:0:66521:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=25 00000080:00200000:1.0:1787826267.225508:0:66521:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:1.0:1787826267.225530:0:66521:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 3 00000080:00200000:1.0:1787826267.225535:0:66521:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), file ffff907d91e9a900, open_flags 8041, mode 81b6 opened 0 00000080:00200000:1.0:1787826267.225544:0:66521:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=open|creat 00000002:00010000:1.0:1787826267.225565:0:66521:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204b.sanity-flr,[0x200000404:0x201:0x0]) in obj [0x200000404:0x1ff:0x0], intent: open|creat flags 0100102 00010000:00010000:1.0:1787826267.230380:0:66521:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b769 lrc: 3/0,1 mode: --/CW res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66521 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.230400:0:66521:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b769 lrc: 3/0,1 mode: --/CW res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66521 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826267.230417:0:66521:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826267.230430:0:66521:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a3840 x1874670011407232/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/67896 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826267.230598:0:66521:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b769 lrc: 3/0,1 mode: --/CW res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66521 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826267.231372:0:66521:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:66521:1874670011407232:192.168.203.104@tcp:101:dd.0 00000100:00100000:1.0:1787826267.232075:0:66521:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe80c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826267.240355:0:66521:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:66521:1874670011407232:192.168.203.104@tcp:101:dd.0 00010000:00010000:1.0:1787826267.240380:0:66521:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b769 lrc: 4/0,1 mode: --/CW res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa6e expref: -99 pid: 66521 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.240401:0:66521:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b769 lrc: 4/0,1 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa6e expref: -99 pid: 66521 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.240410:0:66521:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b769 lrc: 4/0,1 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa6e expref: -99 pid: 66521 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.240420:0:66521:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b769 lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa6e expref: -99 pid: 66521 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.240431:0:66521:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b769 lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa6e expref: -99 pid: 66521 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.240445:0:66521:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b769 lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa6e expref: -99 pid: 66521 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.240453:0:66521:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b769 lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa6e expref: -99 pid: 66521 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.240460:0:66521:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b769 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa6e expref: -99 pid: 66521 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826267.240476:0:66521:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x2b, status=0 req@ffff907d908a3840 x1874670011407232/t8589942958(8589942958) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/1160 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826267.240490:0:66521:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a3840 x1874670011407232/t8589942958(8589942958) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/1160 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826267.240578:0:66521:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b769 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa6e expref: -99 pid: 66521 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826267.240592:0:66521:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b769 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa6e expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.240603:0:66521:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/512 (0 0) 00000080:00200000:1.0:1787826267.240835:0:66521:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x200000404:0x1ff:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1787826267.240888:0:66521:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x200:0x0](ffff907d93313248): rc = 0 00000080:00200000:1.0:1787826267.240903:0:66521:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x200:0x0]: layout version change: 4294967294 -> 5 00000080:00010000:1.0:1787826267.240909:0:66521:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) for lock 0xe5f736940357b769 00000080:00010000:1.0:1787826267.240920:0:66521:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) 00000080:00200008:1.0:1787826267.240933:0:66521:0:(file.c:1090:ll_file_open()) START file f204b.sanity-flr:[0x200000404:0x200:0x0](ffff907d93313248), flags 100101 00000002:00100000:1.0:1787826267.240953:0:66521:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a3840 x1874670011407232/t8589942958(8589942958) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1088/1160 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1787826267.240989:0:66521:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b769 from it ffff907da56c2420 00010000:00010000:1.0:1787826267.240994:0:66521:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b769 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fa6e expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.241004:0:66521:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b769 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fa6e expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826267.241025:0:66521:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248), inode mode 81a4 mask 42 00000080:00200008:0.0:1787826267.253705:0:66521:0:(file.c:2652:do_file_write_iter()) START file f204b.sanity-flr:[0x200000404:0x200:0x0], ppos: 0, count: 2097152 00000080:00200000:0.0:1787826267.253719:0:66521:0:(file.c:2679:do_file_write_iter()) switching to DIO 00000080:00200000:0.0:1787826267.253725:0:66521:0:(file.c:2090:ll_file_io_generic()) f204b.sanity-flr: write ppos: 0, bytes: 2097152 00000080:00200000:0.0:1787826267.253738:0:66521:0:(file.c:1848:ll_io_set_mirror()) f204b.sanity-flr: desiginated mirror: 0 00000020:00200000:0.0:1787826267.253747:0:66521:0:(cl_io.c:218:cl_io_rw_init()) header@000000008ae07a1f[0x110, 2, [0x200000404:0x200:0x0] hash] 00000020:00200000:0.0:1787826267.253749:0:66521:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [0, 2097152) 0 0 00000080:00200000:0.0:1787826267.253754:0:66521:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826267.253770:0:66521:0:(lov_io.c:571:lov_io_mirror_init()) [0x200000404:0x200:0x0] need write intent for [0, 2097152) 00000020:00200000:0.0:1787826267.253775:0:66521:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1787826267.253782:0:66521:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 5 need write layout 1, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1787826267.253786:0:66521:0:(vvp_io.c:371:vvp_io_fini()) [0x200000404:0x200:0x0] write layout, type 2 [0x0, 0x200000) 00000080:00010000:0.0:1787826267.253796:0:66521:0:(file.c:7076:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000404:0x200:0x0](ffff907d93313248) 00000002:00010000:0.0:1787826267.253808:0:66521:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x200:0x0]) in obj [0x200000404:0x200:0x0], intent: layout flags 02 00010000:00010000:0.0:1787826267.253830:0:66521:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b769 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0fa6e expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826267.253849:0:66521:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x200:0x0]: losing layout lock 00010000:00010000:0.0:1787826267.253893:0:66521:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b769 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0fa6e expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.253901:0:66521:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826267.253904:0:66521:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b769 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0fa6e expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.253942:0:66521:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 3/0,1 mode: --/EX res: [0x200000404:0x200:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66521 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.253950:0:66521:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 3/0,1 mode: --/EX res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66521 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826267.253960:0:66521:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826267.253973:0:66521:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908470c0 x1874670011407488/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/2048 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826267.253983:0:66521:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 3/0,1 mode: --/EX res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66521 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826267.254003:0:66521:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:66521:1874670011407488:192.168.203.104@tcp:101:dd.0 00000100:00100000:0.0:1787826267.254197:0:66521:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd0c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826267.270596:0:66521:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589942959 gen 1 00000100:00100000:0.0:1787826267.271408:0:66521:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d908a3840 x1874670011407232/t8589942958(8589942958) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1088/1160 e 0 to 0 dl 1787826285 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1787826267.271424:0:66521:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:66521:1874670011407488:192.168.203.104@tcp:101:dd.0 00010000:00010000:0.0:1787826267.271455:0:66521:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 4/0,1 mode: --/EX res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.271467:0:66521:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 4/0,1 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.271478:0:66521:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 4/0,1 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.271495:0:66521:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 4/0,1 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.271522:0:66521:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 5/1,1 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.271530:0:66521:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 5/1,1 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.271536:0:66521:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826267.271545:0:66521:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x400 disposition=0x1, status=0 req@ffff907d908470c0 x1874670011407488/t8589942959(8589942959) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/896 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826267.271733:0:66521:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826267.271745:0:66521:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.271758:0:66521:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/512 (0 0) 00000080:00200000:1.0:1787826267.271764:0:66521:0:(namei.c:952:ll_intent_lock()) intent lock 1024 on i1 [0x200000404:0x200:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1787826267.271785:0:66521:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) for lock 0xe5f736940357b770 00000080:00010000:1.0:1787826267.271793:0:66521:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b770 from it ffff9dd14186fbb8 00010000:00010000:1.0:1787826267.271796:0:66521:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.271809:0:66521:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826267.271826:0:66521:0:(file.c:2240:ll_file_io_generic()) f204b.sanity-flr: 2 io complete with rc: 0, result: 0, restart: 1 00000080:00200000:1.0:1787826267.271830:0:66521:0:(file.c:2247:ll_file_io_generic()) f204b.sanity-flr: restart write from ppos=0 bytes=2097152 retries=999 ret=0: rc = 0 00000080:00200000:1.0:1787826267.271837:0:66521:0:(file.c:1848:ll_io_set_mirror()) f204b.sanity-flr: desiginated mirror: 0 00000020:00200000:1.0:1787826267.271854:0:66521:0:(cl_io.c:218:cl_io_rw_init()) header@000000008ae07a1f[0x110, 2, [0x200000404:0x200:0x0] hash] 00000020:00200000:1.0:1787826267.271856:0:66521:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [0, 2097152) 0 0 00000080:00200000:1.0:1787826267.271862:0:66521:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 5 restore needed 0 00010000:00010000:1.0:1787826267.271877:0:66521:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826267.271887:0:66521:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:1.0:1787826267.271895:0:66521:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x200:0x0](000000001c1f1433) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826267.271931:0:66521:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826267.271940:0:66521:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826267.273444:0:66521:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826267.273461:0:66521:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826267.273465:0:66521:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826267.273470:0:66521:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1787826267.273507:0:66521:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:1.0:1787826267.273539:0:66521:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1787826267.273563:0:66521:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826267.276062:0:66521:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:1.0:1787826267.276087:0:66521:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x200:0x0](000000001c1f1433) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826267.276135:0:66521:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826267.276145:0:66521:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826267.276148:0:66521:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826267.276152:0:66521:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1787826267.276187:0:66521:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:1.0:1787826267.276206:0:66521:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826267.276296:0:66521:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x200:0x0]: layout version change: 4294967294 -> 6 00010000:00010000:1.0:1787826267.276792:0:66521:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826267.276804:0:66521:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00020000:00200000:1.0:1787826267.276826:0:66521:0:(lov_io.c:311:lov_io_mirror_write_intent()) [0x200000404:0x200:0x0]there are 0 components to be staled to modify file extent [0x0, 0x4100000), iot: 2 00020000:00200000:1.0:1787826267.276830:0:66521:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826267.276832:0:66521:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00000080:00200000:1.0:1787826267.276847:0:66521:0:(file.c:2170:ll_file_io_generic()) Range lock [0, 511] 00020000:00200000:1.0:1787826267.276856:0:66521:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 2097152, 1048576 00020000:00200000:1.0:1787826267.276859:0:66521:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1787826267.276872:0:66521:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 255] 00000008:00010000:1.0:1787826267.276884:0:66521:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1787826267.276888:0:66521:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c32ab0, osc lock ffff907d962f5248, flags 0x0 00000080:00200000:1.0:1787826267.276898:0:66521:0:(vvp_io.c:1339:vvp_io_write_start()) f204b.sanity-flr: write [0, 1048576) append 0 append lockpos 0 00000002:00010000:1.0:1787826267.276926:0:66521:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x0:0x0:0x0]) in obj [0x200000404:0x200:0x0], intent: getxattr flags 0 00010000:00010000:1.0:1787826267.276982:0:66521:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b777 lrc: 3/1,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66521 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.276999:0:66521:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b777 lrc: 3/1,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66521 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.277009:0:66521:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b777 lrc: 3/1,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66521 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826267.277035:0:66521:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:66521:1874670011407616:192.168.203.104@tcp:101:dd.0 00000100:00100000:1.0:1787826267.277125:0:66521:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe80c0 going to sleep for 13 seconds 00000100:00100000:3.0:1787826267.279769:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826267.280081:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025485184 00000100:00100000:2.0:1787826267.280094:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025485184 00000100:00100000:2.0:1787826267.280165:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7133 00000100:00100000:2.0:1787826267.280187:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83526940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:36728:x1874670025485184:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826267.280211:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b6f2 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x308:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc0f861 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826267.280332:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826267.280338:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83526940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:36728:x1874670025485184:12345-192.168.203.104@tcp:104: Request processed in 155us (585us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826267.280347:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7133 00010000:00010000:0.0:1787826267.281273:0:60472:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826267.281287:0:60472:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b6f2 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x308:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc0f861 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.281300:0:60472:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883bc00 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826267.281319:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b6f2 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x308:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x428400010000 nid: local remote: 0x7918213f4fc0f861 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.281389:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b6f2 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x308:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc0f861 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.281398:0:60472:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826267.281439:0:60472:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b6f2 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x308:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc0f861 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.281449:0:60472:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b6f2 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x308:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc0f861 expref: -99 pid: 65912 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.281484:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826267.281487:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826267.281491:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826267.281498:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011407744:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826267.289972:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011407744:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826267.290323:0:66521:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:66521:1874670011407616:192.168.203.104@tcp:101:dd.0 00010000:00010000:1.0:1787826267.290355:0:66521:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b777 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa98 expref: -99 pid: 66521 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.290563:0:66521:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b777 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa98 expref: -99 pid: 66521 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.290580:0:66521:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b777 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa98 expref: -99 pid: 66521 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826267.290600:0:66521:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x80 disposition=0x0, status=0 req@ffff907d908a0b40 x1874670011407616/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 568/1648 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826267.290801:0:66521:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b777 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa98 expref: -99 pid: 66521 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.290822:0:66521:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/512 (0 0) 00000080:00200000:1.0:1787826267.290830:0:66521:0:(namei.c:952:ll_intent_lock()) intent lock 128 on i1 [0x200000404:0x200:0x0] suppgids 0 -1: rc 0 00000080:00010000:2.0:1787826267.291002:0:66521:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) for lock 0xe5f736940357b777 00000080:00010000:2.0:1787826267.291014:0:66521:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b777 from it ffff9dd14186f8e8 00010000:00010000:2.0:1787826267.291019:0:66521:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b777 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fa98 expref: -99 pid: 66521 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.291030:0:66521:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b777 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fa98 expref: -99 pid: 66521 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826267.291205:0:66521:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248), size=1048576 (max 335544320), offset=0=0x0, pages 256 (max 81920), lockless, parallel, hybrid 00000020:00200000:2.0:1787826267.291580:0:66521:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1787826267.291585:0:66521:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 1048576 00000020:00010000:2.0:1787826267.291601:0:66521:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b1038 (ffff907d884a7878/0) at cl_lock_release():245 00000020:00010000:2.0:1787826267.291631:0:66521:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b1038 (ffff907d884a7878/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.291637:0:66521:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c32ab0 (ffff907da5608758/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.291641:0:66521:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b1038 (ffff907d884a7878/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826267.291645:0:66521:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c32ab0 (ffff907d884a7878/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826267.291654:0:66521:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00020000:00200000:2.0:1787826267.291668:0:66521:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 1048576 chunk: [1048576, 2097152) 2097152, 1048576 00020000:00200000:2.0:1787826267.291672:0:66521:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00000008:00010000:2.0:1787826267.291699:0:65304:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907da56f8000 00000008:00100000:2.0:1787826267.292573:0:65304:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d9114b480 - object 0x240000400:804 offset 0<>1048576 00000100:00100000:3.0:1787826267.292693:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011407872:192.168.203.104@tcp:4:dd.0 00000080:00200000:2.0:1787826267.292957:0:66521:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [256, 511] 00000008:00010000:2.0:1787826267.292975:0:66521:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1787826267.292982:0:66521:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322b30, osc lock ffff907d830a3380, flags 0x0 00000080:00200000:2.0:1787826267.292994:0:66521:0:(vvp_io.c:1339:vvp_io_write_start()) f204b.sanity-flr: write [1048576, 2097152) append 0 append lockpos 0 00000080:00200000:2.0:1787826267.293009:0:66521:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248), size=1048576 (max 335544320), offset=1048576=0x100000, pages 256 (max 81920), lockless, parallel, hybrid 00000020:00200000:2.0:1787826267.293538:0:66521:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1787826267.293543:0:66521:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 2097152 00000020:00010000:2.0:1787826267.293551:0:66521:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b1038 (ffff907d884a7878/0) at cl_lock_release():245 00000020:00010000:2.0:1787826267.293554:0:66521:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b1038 (ffff907d884a7878/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.293558:0:66521:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322b30 (ffff907d884a7998/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.293562:0:66521:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b1038 (ffff907d884a7878/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826267.293564:0:66521:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322b30 (ffff907d884a7878/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826267.293572:0:66521:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1787826267.293580:0:66521:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000008:00010000:2.0:1787826267.293597:0:65304:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907da56f8180 00000008:00100000:2.0:1787826267.294538:0:65304:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d91148780 - object 0x240000400:805 offset 1048576<>2097152 00000100:00100000:3.0:1787826267.294708:0:2351:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1787826267.294738:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011408000:192.168.203.104@tcp:4:dd.0 00000100:00100000:3.0:1787826267.350651:0:2351:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff907d986d0000: committing for last_committed 21474840491 gen 3 00000008:00010000:3.0:1787826267.350747:0:2351:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907da56f8000 00000020:00200000:3.0:1787826267.350756:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:3.0:1787826267.350760:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:3.0:1787826267.350761:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:3.0:1787826267.350763:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:3.0:1787826267.350765:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:3.0:1787826267.350766:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:3.0:1787826267.350768:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:3.0:1787826267.350769:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:3.0:1787826267.350771:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:3.0:1787826267.350773:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:3.0:1787826267.350774:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:3.0:1787826267.350775:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:3.0:1787826267.350777:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:3.0:1787826267.350778:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:3.0:1787826267.350780:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:3.0:1787826267.350896:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:3.0:1787826267.350900:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:3.0:1787826267.350902:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:3.0:1787826267.350904:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:3.0:1787826267.350906:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:3.0:1787826267.350908:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:3.0:1787826267.350911:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:3.0:1787826267.350913:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:3.0:1787826267.350914:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:3.0:1787826267.350916:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:3.0:1787826267.350918:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:3.0:1787826267.350919:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:3.0:1787826267.350921:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:3.0:1787826267.350923:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:3.0:1787826267.350924:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:3.0:1787826267.350926:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:3.0:1787826267.350928:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:3.0:1787826267.350930:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:3.0:1787826267.350932:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:3.0:1787826267.350934:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:3.0:1787826267.350936:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:3.0:1787826267.350937:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:3.0:1787826267.350939:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:3.0:1787826267.350941:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:3.0:1787826267.350943:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:3.0:1787826267.350945:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:3.0:1787826267.350947:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:3.0:1787826267.350949:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:3.0:1787826267.350951:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:3.0:1787826267.350952:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:3.0:1787826267.350954:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:3.0:1787826267.350955:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:3.0:1787826267.350956:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:3.0:1787826267.350958:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:3.0:1787826267.350959:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:3.0:1787826267.350961:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:3.0:1787826267.350962:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:3.0:1787826267.350963:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:3.0:1787826267.350966:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:3.0:1787826267.350967:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:3.0:1787826267.350968:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:3.0:1787826267.350970:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:3.0:1787826267.350972:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:3.0:1787826267.350973:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:3.0:1787826267.350975:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:3.0:1787826267.350976:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:3.0:1787826267.350978:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:3.0:1787826267.350979:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:3.0:1787826267.350981:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:3.0:1787826267.350982:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:3.0:1787826267.350984:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:3.0:1787826267.350985:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:3.0:1787826267.350987:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:3.0:1787826267.350988:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:3.0:1787826267.350990:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:3.0:1787826267.350992:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:3.0:1787826267.350994:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:3.0:1787826267.350996:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:3.0:1787826267.350997:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:3.0:1787826267.350999:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:3.0:1787826267.351001:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:3.0:1787826267.351002:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:3.0:1787826267.351004:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:3.0:1787826267.351005:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:3.0:1787826267.351007:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:3.0:1787826267.351009:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:3.0:1787826267.351010:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:3.0:1787826267.351012:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:3.0:1787826267.351013:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:3.0:1787826267.351015:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:3.0:1787826267.351017:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:3.0:1787826267.351019:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:3.0:1787826267.351020:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:3.0:1787826267.351022:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:3.0:1787826267.351023:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:3.0:1787826267.351024:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:3.0:1787826267.351026:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:3.0:1787826267.351027:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:3.0:1787826267.351028:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:3.0:1787826267.351030:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:3.0:1787826267.351031:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:3.0:1787826267.351033:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:3.0:1787826267.351034:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:3.0:1787826267.351036:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:3.0:1787826267.351037:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:3.0:1787826267.351039:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:3.0:1787826267.351041:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:3.0:1787826267.351043:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:3.0:1787826267.351045:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:3.0:1787826267.351047:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:3.0:1787826267.351048:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:3.0:1787826267.351050:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:3.0:1787826267.351051:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:3.0:1787826267.351053:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:3.0:1787826267.351055:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:3.0:1787826267.351056:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:3.0:1787826267.351058:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:3.0:1787826267.351060:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:3.0:1787826267.351061:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:3.0:1787826267.351063:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:3.0:1787826267.351064:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:3.0:1787826267.351066:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:3.0:1787826267.351068:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:3.0:1787826267.351070:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:3.0:1787826267.351072:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:3.0:1787826267.351073:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:3.0:1787826267.351075:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:3.0:1787826267.351076:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:3.0:1787826267.351078:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:3.0:1787826267.351079:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:3.0:1787826267.351081:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:3.0:1787826267.351376:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:3.0:1787826267.351379:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:3.0:1787826267.351381:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:3.0:1787826267.351382:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:3.0:1787826267.351384:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:3.0:1787826267.351386:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:3.0:1787826267.351387:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:3.0:1787826267.351389:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:3.0:1787826267.351390:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:3.0:1787826267.351392:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:3.0:1787826267.351394:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:3.0:1787826267.351396:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:3.0:1787826267.351397:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:3.0:1787826267.351399:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:3.0:1787826267.351401:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:3.0:1787826267.351403:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:3.0:1787826267.351404:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:3.0:1787826267.351406:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:3.0:1787826267.351407:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:3.0:1787826267.351409:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:3.0:1787826267.351410:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:3.0:1787826267.351411:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:3.0:1787826267.351413:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:3.0:1787826267.351415:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:3.0:1787826267.351417:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:3.0:1787826267.351419:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:3.0:1787826267.351421:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:3.0:1787826267.351422:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:3.0:1787826267.351424:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:3.0:1787826267.351425:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:3.0:1787826267.351426:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:3.0:1787826267.351428:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:3.0:1787826267.351429:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:3.0:1787826267.351431:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:3.0:1787826267.351433:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:3.0:1787826267.351435:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:3.0:1787826267.351437:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:3.0:1787826267.351439:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:3.0:1787826267.351441:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:3.0:1787826267.351442:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:3.0:1787826267.351444:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:3.0:1787826267.351445:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:3.0:1787826267.351447:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:3.0:1787826267.351449:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:3.0:1787826267.351451:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:3.0:1787826267.351452:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:3.0:1787826267.351454:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:3.0:1787826267.351456:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:3.0:1787826267.351458:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:3.0:1787826267.351459:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:3.0:1787826267.351461:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:3.0:1787826267.351463:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:3.0:1787826267.351464:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:3.0:1787826267.351466:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:3.0:1787826267.351467:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:3.0:1787826267.351469:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:3.0:1787826267.351471:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:3.0:1787826267.351473:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:3.0:1787826267.351475:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:3.0:1787826267.351476:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:3.0:1787826267.351478:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:3.0:1787826267.351480:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:3.0:1787826267.351481:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:3.0:1787826267.351483:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:3.0:1787826267.351484:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:3.0:1787826267.351486:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:3.0:1787826267.351489:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:3.0:1787826267.351490:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:3.0:1787826267.351492:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:3.0:1787826267.351493:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:3.0:1787826267.351495:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:3.0:1787826267.351497:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:3.0:1787826267.351498:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:3.0:1787826267.351500:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:3.0:1787826267.351502:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:3.0:1787826267.351503:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:3.0:1787826267.351504:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:3.0:1787826267.351506:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:3.0:1787826267.351507:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:3.0:1787826267.351521:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:3.0:1787826267.351524:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:3.0:1787826267.351526:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:3.0:1787826267.351528:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:3.0:1787826267.351530:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:3.0:1787826267.351532:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:3.0:1787826267.351534:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:3.0:1787826267.351535:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:3.0:1787826267.351538:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:3.0:1787826267.351540:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:3.0:1787826267.351541:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:3.0:1787826267.351543:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:3.0:1787826267.351545:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:3.0:1787826267.351546:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:3.0:1787826267.351548:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:3.0:1787826267.351550:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:3.0:1787826267.351551:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:3.0:1787826267.351554:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:3.0:1787826267.351556:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:3.0:1787826267.351557:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:3.0:1787826267.351559:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:3.0:1787826267.351561:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:3.0:1787826267.351563:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:3.0:1787826267.351565:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:3.0:1787826267.351567:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:3.0:1787826267.351568:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:3.0:1787826267.351570:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:3.0:1787826267.351572:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:3.0:1787826267.351574:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:3.0:1787826267.351576:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:3.0:1787826267.351577:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:3.0:1787826267.351579:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:3.0:1787826267.351581:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:3.0:1787826267.351583:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:3.0:1787826267.351585:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:3.0:1787826267.351587:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:3.0:1787826267.351589:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:3.0:1787826267.351591:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:3.0:1787826267.351593:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:3.0:1787826267.351594:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:3.0:1787826267.351596:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:3.0:1787826267.351598:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:3.0:1787826267.351600:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:3.0:1787826267.351602:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:3.0:1787826267.351603:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:3.0:1787826267.351636:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:3.0:1787826267.351639:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:3.0:1787826267.351641:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:3.0:1787826267.351643:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:3.0:1787826267.351645:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:3.0:1787826267.351646:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:3.0:1787826267.352384:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:3.0:1787826267.352398:0:2351:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907da56f8000 00000100:00100000:3.0:1787826267.352436:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011407872:192.168.203.104@tcp:4:dd.0 00000100:00100000:2.0:1787826267.373539:0:2351:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff907d986d0000: committing for last_committed 21474840492 gen 3 00000008:00010000:2.0:1787826267.373624:0:2351:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907da56f8180 00000020:00200000:2.0:1787826267.373630:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1787826267.373633:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1787826267.373634:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1787826267.373636:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1787826267.373637:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1787826267.373639:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1787826267.373640:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1787826267.373642:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1787826267.373643:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1787826267.373644:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1787826267.373646:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1787826267.373647:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1787826267.373648:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1787826267.373649:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1787826267.373650:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1787826267.373652:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1787826267.373653:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1787826267.373654:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1787826267.373655:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1787826267.373656:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1787826267.373657:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1787826267.373659:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1787826267.373660:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1787826267.373661:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1787826267.373662:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1787826267.373663:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1787826267.373664:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1787826267.373665:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1787826267.373667:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1787826267.373668:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1787826267.373669:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1787826267.373670:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1787826267.373671:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1787826267.373672:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1787826267.373674:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1787826267.373675:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1787826267.373676:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1787826267.373677:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1787826267.373679:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1787826267.373680:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1787826267.373681:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1787826267.373682:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1787826267.373683:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1787826267.373685:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1787826267.373686:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1787826267.373687:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1787826267.373688:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1787826267.373689:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1787826267.373690:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1787826267.373691:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1787826267.373693:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1787826267.373694:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1787826267.373695:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1787826267.373696:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1787826267.373697:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1787826267.373698:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1787826267.373700:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1787826267.373701:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1787826267.373702:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1787826267.373703:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1787826267.373704:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1787826267.373705:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1787826267.373706:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1787826267.373707:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1787826267.373708:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1787826267.373710:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1787826267.373711:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1787826267.373712:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1787826267.373714:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1787826267.373716:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1787826267.373721:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1787826267.373723:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1787826267.373724:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1787826267.373725:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1787826267.373727:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1787826267.373729:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1787826267.373730:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1787826267.373731:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1787826267.373733:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1787826267.373734:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1787826267.373735:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1787826267.373737:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1787826267.373738:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1787826267.373740:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1787826267.373741:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1787826267.373743:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1787826267.373744:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1787826267.373746:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1787826267.373747:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1787826267.373749:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1787826267.373750:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1787826267.373752:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1787826267.373753:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1787826267.373755:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1787826267.373756:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1787826267.373758:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1787826267.373760:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1787826267.373761:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1787826267.373763:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1787826267.373765:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1787826267.373766:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1787826267.373768:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1787826267.373770:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1787826267.373772:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1787826267.373774:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1787826267.373776:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1787826267.373777:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1787826267.373778:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1787826267.373780:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1787826267.373785:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1787826267.373786:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1787826267.373787:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1787826267.373789:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1787826267.373791:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1787826267.373792:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1787826267.373793:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1787826267.373794:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1787826267.373796:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1787826267.373797:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1787826267.373801:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1787826267.373802:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1787826267.373803:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1787826267.373805:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1787826267.373806:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1787826267.373807:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1787826267.373809:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1787826267.373810:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1787826267.373812:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1787826267.373813:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1787826267.373817:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1787826267.373818:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1787826267.373820:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1787826267.373821:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1787826267.373823:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1787826267.373824:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1787826267.373826:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1787826267.373827:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1787826267.373828:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1787826267.373830:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1787826267.373831:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1787826267.373832:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1787826267.373834:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1787826267.373835:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1787826267.373836:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1787826267.373837:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1787826267.373839:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1787826267.373840:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1787826267.373841:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1787826267.373843:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1787826267.373844:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1787826267.373846:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1787826267.373847:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1787826267.373848:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1787826267.373849:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1787826267.373851:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1787826267.373852:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1787826267.373853:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1787826267.373855:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1787826267.373856:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1787826267.373857:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1787826267.373858:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1787826267.373860:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1787826267.373861:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1787826267.373862:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1787826267.373864:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1787826267.373865:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1787826267.373867:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1787826267.373868:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1787826267.373870:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1787826267.373871:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1787826267.373873:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1787826267.373874:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1787826267.373875:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1787826267.373876:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1787826267.373878:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1787826267.373879:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1787826267.373880:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1787826267.373882:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1787826267.373883:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1787826267.373884:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1787826267.373885:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1787826267.373887:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1787826267.373888:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1787826267.373889:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1787826267.373891:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1787826267.373892:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1787826267.373893:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1787826267.373894:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1787826267.373896:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1787826267.373897:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1787826267.373898:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1787826267.373900:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1787826267.373903:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1787826267.373905:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1787826267.373906:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1787826267.373907:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1787826267.373908:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1787826267.373910:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1787826267.373912:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1787826267.373913:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1787826267.373914:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1787826267.373915:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1787826267.373917:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1787826267.373918:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1787826267.373919:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1787826267.373920:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1787826267.373922:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1787826267.373923:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1787826267.373924:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1787826267.373925:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1787826267.373927:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1787826267.373928:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1787826267.373929:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1787826267.373930:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1787826267.373932:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1787826267.373933:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1787826267.373934:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1787826267.373936:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1787826267.373937:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1787826267.373938:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1787826267.373939:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1787826267.373941:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1787826267.373942:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1787826267.373943:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1787826267.373944:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1787826267.373946:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1787826267.373947:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1787826267.373948:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1787826267.373949:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1787826267.373951:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1787826267.373952:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1787826267.373954:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1787826267.373955:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1787826267.373956:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1787826267.373957:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1787826267.373959:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1787826267.373960:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1787826267.373961:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1787826267.373962:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1787826267.373964:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1787826267.373965:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1787826267.373966:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1787826267.373967:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1787826267.373969:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1787826267.373970:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1787826267.373971:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1787826267.373973:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1787826267.373974:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1787826267.373975:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1787826267.373977:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1787826267.373978:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1787826267.373979:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1787826267.373981:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1787826267.373982:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1787826267.373983:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1787826267.373985:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1787826267.374071:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1787826267.374092:0:2351:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907da56f8180 00000100:00100000:2.0:1787826267.374120:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011408000:192.168.203.104@tcp:4:dd.0 00000080:00200000:0.0:1787826267.374156:0:66521:0:(file.c:2207:ll_file_io_generic()) Range unlock [0, 511] 00000080:00200000:0.0:1787826267.374183:0:66521:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 6 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1787826267.374190:0:66521:0:(file.c:2240:ll_file_io_generic()) f204b.sanity-flr: 2 io complete with rc: 0, result: 2097152, restart: 0 00000080:00200000:0.0:1787826267.374210:0:66521:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x200:0x0] updating i_size 2097152 i_blocks 4096 00000020:00200000:0.0:1787826267.374215:0:66521:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1787826267.374226:0:66521:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 2097152 00000080:00200008:0.0:1787826267.374276:0:66521:0:(file.c:479:ll_file_release()) START file f204b.sanity-flr:[0x200000404:0x200:0x0](ffff907d93313248), flags 100001 00000002:00100000:0.0:1787826267.374358:0:66521:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d908a3840 x1874670011407232/t8589942958(8589942958) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1088/1160 e 0 to 0 dl 1787826285 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1787826267.374372:0:66521:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1787826267.374378:0:66521:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90844780 x1874670011408256/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1787826267.374406:0:66521:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:66521:1874670011408256:192.168.203.104@tcp:35:dd.0 00000100:00100000:0.0:1787826267.374529:0:66521:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd0c0 going to sleep for 11 seconds 00000100:00100000:0.0:1787826267.376931:0:66521:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:66521:1874670011408256:192.168.203.104@tcp:35:dd.0 00000002:00100000:0.0:1787826267.376973:0:66521:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d908a3840 x1874670011407232/t8589942958(8589942958) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1088/1160 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826267.394080:0:66522:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826267.394119:0:66522:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826267.394144:0:66522:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826267.394163:0:66522:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1787826267.394178:0:66522:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826267.394188:0:66522:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826267.394192:0:66522:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:1.0:1787826267.394201:0:66522:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd140c8fbf8 00010000:00010000:1.0:1787826267.394206:0:66522:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826267.394217:0:66522:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1787826267.394235:0:66522:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826267.394244:0:66522:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=27 00000080:00200000:1.0:1787826267.394250:0:66522:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:1.0:1787826267.394263:0:66522:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), file ffff907d961b5680, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1787826267.394271:0:66522:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=open 00000002:00010000:1.0:1787826267.394288:0:66522:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1ff:0x0], intent: open flags 0304001 00010000:00010000:1.0:1787826267.394357:0:66522:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b77e lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.394366:0:66522:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b77e lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826267.394380:0:66522:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826267.394396:0:66522:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0000 x1874670011408512/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826267.394814:0:66522:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b77e lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826267.394846:0:66522:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66522:1874670011408512:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826267.395144:0:66522:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe80c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826267.401547:0:66522:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66522:1874670011408512:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826267.401583:0:66522:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b77e lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.401600:0:66522:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b77e lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.401634:0:66522:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b77e lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.401644:0:66522:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b77e lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.401655:0:66522:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b77e lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.401663:0:66522:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0a600 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826267.401670:0:66522:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b77e lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.401678:0:66522:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b77e lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.401686:0:66522:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b77e lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.401695:0:66522:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826267.401705:0:66522:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b77e lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.401713:0:66522:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b77e lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826267.401736:0:66522:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908a0000 x1874670011408512/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826267.401753:0:66522:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1ff:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826267.401935:0:66522:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826267.401958:0:66522:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826267.401977:0:66522:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1787826267.401991:0:66522:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1787826267.402000:0:66522:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826267.402006:0:66522:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826267.402010:0:66522:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:1.0:1787826267.402016:0:66522:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd140c8fbf8 00010000:00010000:1.0:1787826267.402020:0:66522:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1787826267.402030:0:66522:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:1.0:1787826267.402047:0:66522:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826267.402056:0:66522:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=267 00000080:00200000:1.0:1787826267.402065:0:66522:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 44 00000080:00200008:1.0:1787826267.402078:0:66522:0:(file.c:1090:ll_file_open()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00000002:00010000:1.0:1787826267.402164:0:66522:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1ff:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:1.0:1787826267.402218:0:66522:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b785 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.402226:0:66522:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b785 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826267.402239:0:66522:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826267.402247:0:66522:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0000 x1874670011408768/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826267.402259:0:66522:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b785 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826267.402281:0:66522:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66522:1874670011408768:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826267.402522:0:66522:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe80c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826267.405720:0:66522:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66522:1874670011408768:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826267.405751:0:66522:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b785 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.405766:0:66522:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b785 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.405773:0:66522:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b785 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.405781:0:66522:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b785 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.405808:0:66522:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b785 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.405816:0:66522:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0bc00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826267.405821:0:66522:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b785 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.405829:0:66522:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b785 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.405837:0:66522:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b785 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.405845:0:66522:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826267.405853:0:66522:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b785 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.405859:0:66522:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b785 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 66522 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826267.405883:0:66522:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffff907d908a0000 x1874670011408768/t8589942961(8589942961) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826267.405898:0:66522:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a0000 x1874670011408768/t8589942961(8589942961) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826267.405912:0:66522:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826267.405939:0:66522:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x1ff:0x0], updating i_size 4096 00000002:00100000:1.0:1787826267.405961:0:66522:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a0000 x1874670011408768/t8589942961(8589942961) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1787826267.406153:0:66522:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8) cmd=c0086915 arg=187f3d0 00000100:00100000:1.0:1787826267.406232:0:66522:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66522:1874670011408896:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826267.406323:0:66522:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe80c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826267.409592:0:66522:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66522:1874670011408896:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826267.409995:0:66522:0:(file.c:479:ll_file_release()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00000002:00100000:1.0:1787826267.410071:0:66522:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d908a0000 x1874670011408768/t8589942961(8589942961) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826285 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826267.410084:0:66522:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826267.410090:0:66522:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a3c00 x1874670011409152/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826267.410114:0:66522:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66522:1874670011409152:192.168.203.104@tcp:35:lfs.0 00000100:00100000:1.0:1787826267.411021:0:66522:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe80c0 going to sleep for 11 seconds 00000100:00100000:1.0:1787826267.414005:0:66522:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66522:1874670011409152:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826267.414053:0:66522:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d908a0000 x1874670011408768/t8589942961(8589942961) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:3.0:1787826267.435293:0:66524:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1787826267.435307:0:66524:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826267.435333:0:66524:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826267.435356:0:66524:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:3.0:1787826267.435374:0:66524:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:3.0:1787826267.435382:0:66524:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826267.435393:0:66524:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826267.435397:0:66524:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:3.0:1787826267.435406:0:66524:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141883cb8 00010000:00010000:3.0:1787826267.435410:0:66524:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:3.0:1787826267.435424:0:66524:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:3.0:1787826267.435902:0:66524:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826267.435923:0:66524:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826267.435933:0:66524:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:3.0:1787826267.435943:0:66524:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:3.0:1787826267.435949:0:66524:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826267.435954:0:66524:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826267.435957:0:66524:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:3.0:1787826267.435961:0:66524:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141883bb0 00010000:00010000:3.0:1787826267.435964:0:66524:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:3.0:1787826267.435971:0:66524:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:3.0:1787826267.435982:0:66524:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:3.0:1787826267.435992:0:66524:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:3.0:1787826267.436015:0:66524:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffd820be45c 00000080:00200008:3.0:1787826267.436031:0:66524:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:1.0:1787826267.445708:0:66528:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826267.445746:0:66528:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826267.445771:0:66528:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1787826267.446032:0:66528:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:1.0:1787826267.446052:0:66528:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826267.446064:0:66528:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826267.446069:0:66528:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:1.0:1787826267.446079:0:66528:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141c0fbf8 00010000:00010000:1.0:1787826267.446149:0:66528:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1787826267.446163:0:66528:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1787826267.446183:0:66528:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826267.446194:0:66528:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=27 00000080:00200000:1.0:1787826267.446203:0:66528:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:1.0:1787826267.446220:0:66528:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), file ffff907d84e73980, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1787826267.446228:0:66528:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=open 00000002:00010000:1.0:1787826267.446245:0:66528:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1ff:0x0], intent: open flags 0304001 00010000:00010000:1.0:1787826267.446429:0:66528:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b78c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66528 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.446439:0:66528:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b78c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66528 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826267.446455:0:66528:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826267.446471:0:66528:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a3c00 x1874670011409408/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826267.446484:0:66528:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b78c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66528 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826267.446522:0:66528:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66528:1874670011409408:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826267.451603:0:66528:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe80c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826267.462067:0:66528:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66528:1874670011409408:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826267.462098:0:66528:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b78c lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66528 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.462115:0:66528:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b78c lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 66528 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.462122:0:66528:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b78c lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 66528 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.462130:0:66528:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b78c lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 66528 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.462139:0:66528:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b78c lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 66528 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.462145:0:66528:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0bc00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826267.462150:0:66528:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b78c lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 66528 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.462157:0:66528:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b78c lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 66528 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.462164:0:66528:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b78c lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 66528 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.462171:0:66528:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826267.462180:0:66528:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b78c lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 66528 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.462187:0:66528:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b78c lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 66528 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826267.462210:0:66528:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908a3c00 x1874670011409408/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826267.462225:0:66528:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1ff:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826267.462543:0:66528:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826267.462566:0:66528:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826267.462584:0:66528:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1787826267.462599:0:66528:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1787826267.462661:0:66528:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826267.462668:0:66528:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826267.462672:0:66528:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:1.0:1787826267.462679:0:66528:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141c0fbf8 00010000:00010000:1.0:1787826267.462682:0:66528:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1787826267.462691:0:66528:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1787826267.462707:0:66528:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826267.462716:0:66528:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=267 00000080:00200000:1.0:1787826267.462726:0:66528:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 44 00000080:00200008:1.0:1787826267.462736:0:66528:0:(file.c:1090:ll_file_open()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00000002:00010000:1.0:1787826267.462752:0:66528:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x1ff:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240304001 00010000:00010000:1.0:1787826267.465031:0:66528:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66528 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.465053:0:66528:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66528 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826267.465079:0:66528:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826267.465098:0:66528:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a3c00 x1874670011409664/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826267.465109:0:66528:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66528 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826267.465145:0:66528:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66528:1874670011409664:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826267.465385:0:66528:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe80c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826267.475358:0:66528:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66528:1874670011409664:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826267.475401:0:66528:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fb01 expref: -99 pid: 66528 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.475413:0:66528:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fb01 expref: -99 pid: 66528 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.475432:0:66528:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fb01 expref: -99 pid: 66528 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.475444:0:66528:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fb01 expref: -99 pid: 66528 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826267.475463:0:66528:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x200002b, status=0 req@ffff907d908a3c00 x1874670011409664/t8589942963(8589942963) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826267.475479:0:66528:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a3c00 x1874670011409664/t8589942963(8589942963) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826267.475491:0:66528:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fb01 expref: -99 pid: 66528 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.475503:0:66528:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/511 (0 0) 00000080:00200000:1.0:1787826267.475521:0:66528:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826267.475547:0:66528:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x1ff:0x0], updating i_size 4096 00000080:00010000:1.0:1787826267.475555:0:66528:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x1ff:0x0](ffff907d9221e3c8) for lock 0xe5f736940357b793 00000080:00010000:1.0:1787826267.475565:0:66528:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b793 from it ffff9dd141c0fbd0 00010000:00010000:1.0:1787826267.475570:0:66528:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fb01 expref: -99 pid: 66528 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.475580:0:66528:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fb01 expref: -99 pid: 66528 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1787826267.475639:0:66528:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a3c00 x1874670011409664/t8589942963(8589942963) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1787826267.475757:0:66528:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8) cmd=c1406916 arg=1ad2a20 00000100:00100000:1.0:1787826267.475848:0:66528:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66528:1874670011409792:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826267.475963:0:66528:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe80c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826267.482005:0:66528:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66528:1874670011409792:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826267.482529:0:66528:0:(file.c:479:ll_file_release()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00010000:00010000:1.0:1787826267.482566:0:66528:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fb01 expref: -99 pid: 66528 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200008:2.0:1787826267.512844:0:66531:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1787826267.512857:0:66531:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826267.512885:0:66531:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826267.512908:0:66531:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1787826267.512928:0:66531:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:2.0:1787826267.512937:0:66531:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826267.512946:0:66531:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826267.512951:0:66531:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:2.0:1787826267.512960:0:66531:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd14195bcb8 00010000:00010000:2.0:1787826267.512965:0:66531:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1787826267.512974:0:66531:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:2.0:1787826267.513093:0:66531:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826267.513102:0:66531:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826267.513109:0:66531:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1787826267.513118:0:66531:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:2.0:1787826267.513125:0:66531:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826267.513129:0:66531:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826267.513133:0:66531:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:2.0:1787826267.513137:0:66531:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd14195bbb0 00010000:00010000:2.0:1787826267.513141:0:66531:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1787826267.513148:0:66531:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:2.0:1787826267.513157:0:66531:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:2.0:1787826267.513166:0:66531:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:2.0:1787826267.513186:0:66531:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7fff6d7d044c 00000080:00200008:2.0:1787826267.513203:0:66531:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:1.0:1787826267.520398:0:66534:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826267.520646:0:66534:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826267.520671:0:66534:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1787826267.520690:0:66534:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:1.0:1787826267.520699:0:66534:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826267.520711:0:66534:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826267.520715:0:66534:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:1.0:1787826267.520724:0:66534:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141c37bf8 00010000:00010000:1.0:1787826267.520728:0:66534:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1787826267.520737:0:66534:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:1.0:1787826267.520755:0:66534:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826267.520764:0:66534:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=27 00000080:00200000:1.0:1787826267.520771:0:66534:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:1.0:1787826267.520785:0:66534:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), file ffff907d961b5380, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1787826267.520792:0:66534:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=open 00000002:00010000:1.0:1787826267.520821:0:66534:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1ff:0x0], intent: open flags 0304001 00010000:00010000:1.0:1787826267.520889:0:66534:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b79a lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 66534 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.520898:0:66534:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b79a lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66534 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826267.520912:0:66534:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826267.520926:0:66534:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0000 x1874670011410048/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826267.520937:0:66534:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b79a lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66534 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826267.520965:0:66534:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66534:1874670011410048:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826267.521212:0:66534:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe80c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826267.527098:0:66534:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66534:1874670011410048:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826267.527133:0:66534:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b79a lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66534 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.527151:0:66534:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b79a lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66534 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.527162:0:66534:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b79a lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66534 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.527170:0:66534:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b79a lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66534 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.527180:0:66534:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b79a lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66534 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.527188:0:66534:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0bc00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826267.527193:0:66534:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b79a lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66534 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.527204:0:66534:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b79a lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66534 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.527212:0:66534:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b79a lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 66534 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.527220:0:66534:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826267.527229:0:66534:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b79a lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66534 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.527236:0:66534:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b79a lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66534 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826267.527257:0:66534:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908a0000 x1874670011410048/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826267.527642:0:66534:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1ff:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826267.528579:0:66534:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826267.529202:0:66534:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826267.529227:0:66534:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1787826267.529246:0:66534:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:1.0:1787826267.529261:0:66534:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826267.529272:0:66534:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826267.529276:0:66534:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:1.0:1787826267.529284:0:66534:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141c37bf8 00010000:00010000:1.0:1787826267.529288:0:66534:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1787826267.529297:0:66534:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:1.0:1787826267.529441:0:66534:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826267.529452:0:66534:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=267 00000080:00200000:1.0:1787826267.530125:0:66534:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 44 00000080:00200008:1.0:1787826267.530140:0:66534:0:(file.c:1090:ll_file_open()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00000080:00200080:1.0:1787826267.530203:0:66534:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8) cmd=c1406916 arg=1c5ba20 00000100:00100000:1.0:1787826267.530295:0:66534:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66534:1874670011410176:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826267.530540:0:66534:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe80c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826267.535684:0:66534:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66534:1874670011410176:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826267.535891:0:66534:0:(file.c:479:ll_file_release()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00010000:00010000:1.0:1787826267.535928:0:66534:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fb01 expref: -99 pid: 66528 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200008:1.0:1787826267.563297:0:66538:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1787826267.563435:0:66538:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826267.563461:0:66538:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826267.563485:0:66538:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1787826267.563520:0:66538:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:1.0:1787826267.563529:0:66538:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826267.563538:0:66538:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826267.563542:0:66538:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:1.0:1787826267.563552:0:66538:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd14186fcb8 00010000:00010000:1.0:1787826267.563556:0:66538:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:1.0:1787826267.563566:0:66538:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:0.0:1787826267.568913:0:66538:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826267.568953:0:66538:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826267.568980:0:66538:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:0.0:1787826267.568999:0:66538:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:0.0:1787826267.569009:0:66538:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826267.569017:0:66538:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826267.569021:0:66538:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826267.569031:0:66538:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd14186fbb0 00010000:00010000:0.0:1787826267.569035:0:66538:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:0.0:1787826267.569044:0:66538:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:0.0:1787826267.569063:0:66538:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:0.0:1787826267.569072:0:66538:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:0.0:1787826267.569222:0:66538:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffc60f720bc 00000080:00200008:0.0:1787826267.569241:0:66538:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:2.0:1787826267.571635:0:66541:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826267.571669:0:66541:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826267.571691:0:66541:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:2.0:1787826267.571709:0:66541:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:2.0:1787826267.571716:0:66541:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826267.571726:0:66541:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826267.571729:0:66541:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:2.0:1787826267.571738:0:66541:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141c07bf8 00010000:00010000:2.0:1787826267.571742:0:66541:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:2.0:1787826267.571753:0:66541:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:2.0:1787826267.571772:0:66541:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826267.571780:0:66541:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=27 00000080:00200000:2.0:1787826267.571786:0:66541:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:2.0:1787826267.571799:0:66541:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), file ffff907d98b6ef00, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1787826267.571818:0:66541:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=open 00000002:00010000:2.0:1787826267.571835:0:66541:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1ff:0x0], intent: open flags 0304001 00010000:00010000:2.0:1787826267.571899:0:66541:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a400/0xe5f736940357b7a1 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 66541 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.571907:0:66541:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a400/0xe5f736940357b7a1 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66541 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826267.571920:0:66541:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826267.571936:0:66541:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d91148780 x1874670011410432/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826267.571947:0:66541:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a400/0xe5f736940357b7a1 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66541 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826267.571972:0:66541:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66541:1874670011410432:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826267.572379:0:66541:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bbf00 going to sleep for 13 seconds 00000100:00100000:3.0:1787826267.583275:0:66541:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66541:1874670011410432:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826267.583678:0:66541:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a400/0xe5f736940357b7a1 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66541 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826267.583696:0:66541:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a400/0xe5f736940357b7a1 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66541 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826267.583704:0:66541:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a400/0xe5f736940357b7a1 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66541 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826267.583712:0:66541:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a400/0xe5f736940357b7a1 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66541 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826267.583722:0:66541:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a400/0xe5f736940357b7a1 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66541 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826267.583728:0:66541:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907da639a400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826267.583734:0:66541:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a400/0xe5f736940357b7a1 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66541 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826267.583741:0:66541:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a400/0xe5f736940357b7a1 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66541 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826267.583749:0:66541:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a400/0xe5f736940357b7a1 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 66541 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826267.583756:0:66541:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826267.583765:0:66541:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a400/0xe5f736940357b7a1 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66541 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826267.583772:0:66541:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a400/0xe5f736940357b7a1 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66541 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826267.583867:0:66541:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d91148780 x1874670011410432/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826267.583885:0:66541:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1ff:0x0] suppgids 0 -1: rc -20 00000080:00200000:3.0:1787826267.584772:0:66541:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826267.584864:0:66541:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826267.584887:0:66541:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:3.0:1787826267.584904:0:66541:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:3.0:1787826267.584913:0:66541:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826267.584921:0:66541:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826267.584925:0:66541:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:3.0:1787826267.584931:0:66541:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141c07bf8 00010000:00010000:3.0:1787826267.584935:0:66541:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:3.0:1787826267.584944:0:66541:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:3.0:1787826267.584960:0:66541:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826267.584968:0:66541:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=267 00000080:00200000:3.0:1787826267.584975:0:66541:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 44 00000080:00200008:3.0:1787826267.584985:0:66541:0:(file.c:1090:ll_file_open()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00000080:00200080:3.0:1787826267.585029:0:66541:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8) cmd=c1406916 arg=19dca20 00000100:00100000:3.0:1787826267.585765:0:66541:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66541:1874670011410560:192.168.203.104@tcp:34:lfs.0 00000100:00100000:3.0:1787826267.585984:0:66541:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7da80 going to sleep for 13 seconds 00000100:00100000:3.0:1787826267.597798:0:66541:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66541:1874670011410560:192.168.203.104@tcp:34:lfs.0 00000080:00200008:2.0:1787826267.600400:0:66541:0:(file.c:479:ll_file_release()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00010000:00010000:2.0:1787826267.600450:0:66541:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fb01 expref: -99 pid: 66528 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200008:0.0:1787826267.631406:0:66545:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1787826267.631419:0:66545:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826267.631445:0:66545:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826267.631468:0:66545:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00010000:00010000:0.0:1787826267.631486:0:66545:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:0.0:1787826267.631495:0:66545:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826267.631505:0:66545:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826267.632173:0:66545:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826267.632188:0:66545:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141c27cb8 00010000:00010000:0.0:1787826267.632193:0:66545:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00010000:00010000:0.0:1787826267.632215:0:66545:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:0.0:1787826267.633154:0:66545:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826267.633176:0:66545:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826267.633191:0:66545:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:0.0:1787826267.633204:0:66545:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:0.0:1787826267.633212:0:66545:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826267.633217:0:66545:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826267.633220:0:66545:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826267.633225:0:66545:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141c27bb0 00010000:00010000:0.0:1787826267.633228:0:66545:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:0.0:1787826267.633237:0:66545:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:0.0:1787826267.633249:0:66545:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:0.0:1787826267.633256:0:66545:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:0.0:1787826267.633276:0:66545:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffddfe212cc 00000080:00200008:0.0:1787826267.633290:0:66545:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:2.0:1787826267.635471:0:66548:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826267.635500:0:66548:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826267.635531:0:66548:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1787826267.635549:0:66548:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:2.0:1787826267.635558:0:66548:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826267.635566:0:66548:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826267.635570:0:66548:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:2.0:1787826267.635579:0:66548:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141cdbbf8 00010000:00010000:2.0:1787826267.635583:0:66548:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1787826267.635593:0:66548:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:2.0:1787826267.635637:0:66548:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826267.635646:0:66548:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=27 00000080:00200000:2.0:1787826267.635653:0:66548:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:2.0:1787826267.635666:0:66548:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), file ffff907d98b6e480, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1787826267.635674:0:66548:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=open 00000002:00010000:2.0:1787826267.635691:0:66548:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1ff:0x0], intent: open flags 0304001 00010000:00010000:2.0:1787826267.635764:0:66548:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639bc00/0xe5f736940357b7a8 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 66548 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.635774:0:66548:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639bc00/0xe5f736940357b7a8 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66548 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826267.635787:0:66548:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826267.635803:0:66548:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d91148b40 x1874670011410816/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826267.635814:0:66548:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639bc00/0xe5f736940357b7a8 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66548 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826267.635841:0:66548:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66548:1874670011410816:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826267.636068:0:66548:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb300 going to sleep for 13 seconds 00000100:00100000:1.0:1787826267.643801:0:66548:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66548:1874670011410816:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826267.643833:0:66548:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639bc00/0xe5f736940357b7a8 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66548 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.643850:0:66548:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639bc00/0xe5f736940357b7a8 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66548 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.643858:0:66548:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639bc00/0xe5f736940357b7a8 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66548 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.643866:0:66548:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639bc00/0xe5f736940357b7a8 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66548 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.643876:0:66548:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639bc00/0xe5f736940357b7a8 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66548 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.643883:0:66548:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907da639bc00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826267.643889:0:66548:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639bc00/0xe5f736940357b7a8 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66548 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.643897:0:66548:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639bc00/0xe5f736940357b7a8 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66548 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.643905:0:66548:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639bc00/0xe5f736940357b7a8 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 66548 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.643914:0:66548:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826267.643923:0:66548:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639bc00/0xe5f736940357b7a8 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66548 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.643930:0:66548:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639bc00/0xe5f736940357b7a8 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66548 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826267.643957:0:66548:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d91148b40 x1874670011410816/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826267.643973:0:66548:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1ff:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826267.644308:0:66548:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826267.644333:0:66548:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826267.644352:0:66548:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1787826267.644366:0:66548:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:1.0:1787826267.644373:0:66548:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826267.644380:0:66548:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826267.644384:0:66548:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:1.0:1787826267.644390:0:66548:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141cdbbf8 00010000:00010000:1.0:1787826267.644394:0:66548:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1787826267.644402:0:66548:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:1.0:1787826267.644418:0:66548:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826267.644426:0:66548:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=267 00000080:00200000:1.0:1787826267.644434:0:66548:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 44 00000080:00200008:1.0:1787826267.644444:0:66548:0:(file.c:1090:ll_file_open()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00000080:00200080:1.0:1787826267.644487:0:66548:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8) cmd=c1406916 arg=92ba20 00000100:00100000:1.0:1787826267.644569:0:66548:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66548:1874670011410944:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826267.644700:0:66548:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe80c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826267.648375:0:66548:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66548:1874670011410944:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826267.648594:0:66548:0:(file.c:479:ll_file_release()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00010000:00010000:1.0:1787826267.648647:0:66548:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fb01 expref: -99 pid: 66528 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1787826267.667424:0:66551:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826267.667467:0:66551:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826267.667492:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:3.0:1787826267.667520:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:3.0:1787826267.667529:0:66551:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826267.667539:0:66551:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826267.667543:0:66551:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:3.0:1787826267.667553:0:66551:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141cdbb58 00010000:00010000:3.0:1787826267.667557:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:3.0:1787826267.667565:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:3.0:1787826267.667583:0:66551:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826267.667592:0:66551:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=25 00000080:00200000:3.0:1787826267.667598:0:66551:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:3.0:1787826267.667757:0:66551:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags=9 00000080:00200000:3.0:1787826267.667763:0:66551:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=getattr 00000002:00010000:3.0:1787826267.667778:0:66551:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1ff:0x0], intent: getattr flags 01 00010000:00010000:3.0:1787826267.667845:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 66551 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826267.667855:0:66551:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66551 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826267.667867:0:66551:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66551 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826267.667899:0:66551:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011411072:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826267.668068:0:66551:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7da80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826267.673165:0:66551:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011411072:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826267.673203:0:66551:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.673224:0:66551:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 4/1,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.673233:0:66551:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 4/1,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.673247:0:66551:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.673260:0:66551:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.673275:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.673289:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.673297:0:66551:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826267.673310:0:66551:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d83524780 x1874670011411072/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/1136 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826267.673323:0:66551:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1787826267.673339:0:66551:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.673350:0:66551:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/512 (0 0) 00000080:00200000:0.0:1787826267.673358:0:66551:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1ff:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1787826267.673428:0:66551:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x200:0x0](ffff907d93313248): rc = 0 00000080:00200000:0.0:1787826267.673448:0:66551:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x200:0x0]: layout version change: 6 -> 6 00000080:00010000:0.0:1787826267.673454:0:66551:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) for lock 0xe5f736940357b7af 00000080:00010000:0.0:1787826267.673468:0:66551:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) 00000080:00010000:0.0:1787826267.673472:0:66551:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b7af from it ffff9dd141cdbb20 00010000:00010000:0.0:1787826267.673476:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.673485:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:0.0:1787826267.673547:0:66551:0:(file.c:6313:ll_getattr_dentry()) START file f204b.sanity-flr:[0x200000404:0x200:0x0](ffff907d93313248), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1787826267.673556:0:66551:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248),name=f204b.sanity-flr 00000002:00010000:0.0:1787826267.673582:0:66551:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x200:0x0]) in obj [0x200000404:0x1ff:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826267.673594:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.673715:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826267.673723:0:66551:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1ff:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826267.673729:0:66551:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) 00000080:00010000:0.0:1787826267.673732:0:66551:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) for lock 0xe5f736940357b7af 00000080:00010000:0.0:1787826267.673736:0:66551:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b7af from it ffff9dd141cdbcb8 00010000:00010000:0.0:1787826267.673739:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.673746:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826267.673768:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/1, layout version 6 restore needed 0 00020000:00200000:0.0:1787826267.673782:0:66551:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826267.673786:0:66551:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:0.0:1787826267.673790:0:66551:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x200:0x0] 00000008:00010000:0.0:1787826267.673813:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1787826267.673817:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d998fd930, osc lock ffff907d96369248, flags 0x400000400001002 00000008:00010000:0.0:1787826267.673825:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1787826267.673827:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d998fd970, osc lock ffff907d963694b8, flags 0x400000400001002 00010000:00010000:0.0:1787826267.673884:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.673895:0:66551:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.673905:0:66551:0:(ldlm_request.c:1139:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff907d90844780 x1874670011411200/t0(0) o101->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1787826267.673962:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1787826267.673982:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011411200:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826267.674115:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 3/1,0 mode: --/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.674125:0:66551:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 3/1,0 mode: --/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [1048576->68157439] (req 1048576->68157439) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.674135:0:66551:0:(ldlm_request.c:1139:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff907d90844b40 x1874670011411328/t0(0) o101->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000020:00200000:0.0:1787826267.674152:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000100:00100000:0.0:1787826267.674166:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011411328:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826267.680460:0:2349:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 6/2,0 mode: --/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [1048576->68157439] (req 1048576->68157439) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.680496:0:2349:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 6/2,0 mode: --/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fbc5 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.680535:0:2349:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fbc5 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.680550:0:2349:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fbc5 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826267.680570:0:2349:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fbc5 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1787826267.680584:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1787826267.680589:0:2349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fbc5 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.680598:0:2349:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fbc5 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.680648:0:2349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fbc5 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.680659:0:2349:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fbc5 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1787826267.680686:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011411328:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826267.680705:0:2349:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.680715:0:2349:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fbbe expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.680724:0:2349:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fbbe expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.680733:0:2349:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fbbe expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826267.680741:0:2349:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 1048576, setting rss=1048576; kms=1048576, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fbbe expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1787826267.680750:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1787826267.680768:0:2349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fbbe expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.680776:0:2349:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fbbe expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.680783:0:2349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fbbe expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.680792:0:2349:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fbbe expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1787826267.680818:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011411200:192.168.203.104@tcp:101:lfs.0 00000080:00200000:3.0:1787826267.681740:0:66551:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x200:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:3.0:1787826267.681749:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da57909e0 (ffff907d884a7878/0) at cl_lock_release():245 00000020:00010000:3.0:1787826267.681753:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da57909e0 (ffff907d884a7878/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826267.681760:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d998fd930 (ffff907d884a7998/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826267.681766:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d998fd970 (ffff907da5608758/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826267.681769:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da57909e0 (ffff907d884a7878/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826267.681772:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d998fd930 (ffff907d884a7878/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826267.681780:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d998fd970 (ffff907d884a7878/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826267.681798:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/1, layout version 6 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787826267.681935:0:66551:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826267.681960:0:66551:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826267.681981:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:3.0:1787826267.681995:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:3.0:1787826267.682003:0:66551:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826267.682011:0:66551:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826267.682015:0:66551:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:3.0:1787826267.682022:0:66551:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141cdbbf8 00010000:00010000:3.0:1787826267.682025:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:3.0:1787826267.682032:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:3.0:1787826267.682045:0:66551:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826267.682052:0:66551:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=25 00000080:00200000:3.0:1787826267.682058:0:66551:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:3.0:1787826267.682062:0:66551:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204b.sanity-flr, flags=265 00000080:00200000:3.0:1787826267.682067:0:66551:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248), inode mode 81a4 mask 46 00000080:00200008:3.0:1787826267.682077:0:66551:0:(file.c:1090:ll_file_open()) START file f204b.sanity-flr:[0x200000404:0x200:0x0](ffff907d93313248), flags 154402 00000002:00010000:3.0:1787826267.682293:0:66551:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x200:0x0]) in obj [0x200000404:0x1ff:0x0], intent: open flags 0100240114403 00010000:00010000:3.0:1787826267.682480:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 66551 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826267.682490:0:66551:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66551 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826267.682503:0:66551:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826267.682527:0:66551:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d835261c0 x1874670011411584/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826267.682538:0:66551:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66551 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826267.682564:0:66551:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011411584:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826267.683162:0:66551:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7db40 going to sleep for 13 seconds 00000100:00100000:2.0:1787826267.687593:0:66551:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011411584:192.168.203.104@tcp:101:lfs.0 00010000:00010000:2.0:1787826267.687746:0:66551:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fbd3 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.687816:0:66551:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fbd3 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.687824:0:66551:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fbd3 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.687839:0:66551:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fbd3 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.687852:0:66551:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fbd3 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.687869:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fbd3 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.687878:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fbd3 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.687886:0:66551:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fbd3 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826267.687898:0:66551:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffff907d835261c0 x1874670011411584/t8589942964(8589942964) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1160 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1787826267.687912:0:66551:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d835261c0 x1874670011411584/t8589942964(8589942964) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1160 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1787826267.687927:0:66551:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fbd3 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1787826267.687941:0:66551:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fbd3 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.687955:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826267.687970:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826267.687978:0:66551:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fbd3 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.687984:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x7918213f4fc0fbd3 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.687991:0:66551:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x7918213f4fc0fbd3 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.688021:0:66551:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:2.0:1787826267.688029:0:66551:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1ff:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1787826267.688108:0:66551:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x200:0x0]: layout version change: 6 -> 6 00000080:00010000:2.0:1787826267.688116:0:66551:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) for lock 0xe5f736940357b7af 00000080:00010000:2.0:1787826267.688125:0:66551:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b7af from it ffff9dd141cdbbf0 00010000:00010000:2.0:1787826267.688128:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826267.688285:0:60483:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1787826267.688296:0:60483:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7918213f4fc0fbd3 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.688326:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000002:00100000:2.0:1787826267.688360:0:66551:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d835261c0 x1874670011411584/t8589942964(8589942964) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1787826267.688582:0:66551:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248), inode mode 81a4 mask 4 00000080:00200000:2.0:1787826267.688593:0:66551:0:(xattr.c:760:ll_xattr_get()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248), xattr lov 00000080:00200008:2.0:1787826267.688660:0:66551:0:(file.c:6313:ll_getattr_dentry()) START file f204b.sanity-flr:[0x200000404:0x200:0x0](ffff907d93313248), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1787826267.688668:0:66551:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248),name=f204b.sanity-flr 00000002:00010000:2.0:1787826267.688692:0:66551:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x200:0x0]) in obj [0x200000404:0x1ff:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826267.688825:0:60483:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d86042200 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826267.688832:0:60483:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x1b rrc: 6 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7918213f4fc0fbd3 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826267.688843:0:60483:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x1b rrc: 6 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7918213f4fc0fbd3 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826267.688852:0:60483:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 3/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x1b rrc: 6 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc0fbd3 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826267.688872:0:60483:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1787826267.688919:0:60483:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 2/0,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x1b rrc: 6 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0fbd3 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826267.688926:0:60483:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787826267.688959:0:60483:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x1b rrc: 6 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0fbd3 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826267.688967:0:60483:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042200/0xe5f736940357b7c4 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x1b rrc: 6 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0fbd3 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826267.688984:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826267.688987:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826267.688990:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826267.689010:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011411712:192.168.203.104@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1787826267.690423:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826267.690449:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1787826267.690461:0:66551:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1ff:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826267.690472:0:66551:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) 00000080:00010000:2.0:1787826267.690476:0:66551:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) for lock 0xe5f736940357b7af 00000080:00010000:2.0:1787826267.690485:0:66551:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b7af from it ffff9dd141cdbcd8 00010000:00010000:2.0:1787826267.690490:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826267.690498:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1787826267.690540:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:2.0:1787826267.690559:0:66551:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826267.690563:0:66551:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1787826267.690568:0:66551:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x200:0x0] 00000008:00010000:2.0:1787826267.690597:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826267.690602:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d870bb870, osc lock ffff907d830a3380, flags 0x400000400001002 00000008:00010000:2.0:1787826267.690681:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826267.690683:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d870bb8b0, osc lock ffff907d830a29c0, flags 0x400000400001002 00010000:00010000:2.0:1787826267.690707:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fbbe expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826267.690720:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fbbe expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1787826267.690733:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826267.690738:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fbbe expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826267.690746:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fbbe expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826267.690822:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fbc5 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826267.690833:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 68157439) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fbc5 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1787826267.690841:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826267.690844:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fbc5 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826267.690852:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fbc5 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1787826267.690861:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826267.690871:0:66551:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x200:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826267.690877:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da57913c0 (ffff907d884a7638/0) at cl_lock_release():245 00000020:00010000:2.0:1787826267.690880:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da57913c0 (ffff907d884a7638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.690883:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d870bb870 (ffff907d884a7488/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.690888:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d870bb8b0 (ffff907da5608368/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.690891:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da57913c0 (ffff907d884a7638/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826267.690895:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d870bb870 (ffff907d884a7638/1) at cl_lock_fini():72 00000020:00010000:2.0:1787826267.690900:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d870bb8b0 (ffff907d884a7638/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826267.690913:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/1, layout version 6 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:2.0:1787826267.690985:0:66551:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248) cmd=c00c66f3 arg=8892a0 00000002:00010000:2.0:1787826267.691008:0:66551:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x200:0x0]) in obj [0x200000404:0x200:0x0], intent: open flags 05004040000002 00010000:00010000:2.0:1787826267.691067:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a200/0xe5f736940357b7cb lrc: 3/1,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 66551 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826267.691075:0:66551:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x81000000001000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a200/0xe5f736940357b7cb lrc: 3/1,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x0 pid: 66551 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826267.691664:0:66551:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826267.691679:0:66551:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114ad00 x1874670011411968/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826267.691692:0:66551:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a200/0xe5f736940357b7cb lrc: 3/1,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x81000000000000 pid: 66551 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826267.691719:0:66551:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011411968:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826267.691971:0:66551:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb840 going to sleep for 13 seconds 00000100:00100000:3.0:1787826267.694704:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011411712:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826267.694858:0:66551:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011411968:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826267.694883:0:66551:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PW ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a200/0xe5f736940357b7cb lrc: 4/1,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc0fbe8 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.694898:0:66551:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a200/0xe5f736940357b7cb lrc: 4/1,0 mode: --/PW res: [0x200000404:0x200:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc0fbe8 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.694912:0:66551:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a200/0xe5f736940357b7cb lrc: 4/1,0 mode: PW/PW res: [0x200000404:0x200:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0fbe8 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.694924:0:66551:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a200/0xe5f736940357b7cb lrc: 4/1,0 mode: PW/PW res: [0x200000404:0x200:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc0fbe8 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.694937:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a200/0xe5f736940357b7cb lrc: 5/1,1 mode: PW/PW res: [0x200000404:0x200:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0fbe8 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.694945:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a200/0xe5f736940357b7cb lrc: 5/1,1 mode: PW/PW res: [0x200000404:0x200:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0fbe8 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.694953:0:66551:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a200/0xe5f736940357b7cb lrc: 4/0,1 mode: PW/PW res: [0x200000404:0x200:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0fbe8 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826267.694966:0:66551:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x600002b, status=0 req@ffff907d9114ad00 x1874670011411968/t8589942965(8589942965) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1160 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1787826267.694985:0:66551:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d9114ad00 x1874670011411968/t8589942965(8589942965) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1160 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826267.695001:0:66551:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a200/0xe5f736940357b7cb lrc: 3/0,1 mode: PW/PW res: [0x200000404:0x200:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc0fbe8 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.695016:0:66551:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 2 res 8589935620/512 (0 0) 00000080:00200000:0.0:1787826267.695024:0:66551:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x200:0x0] suppgids 0 0: rc 0 00000002:00100000:0.0:1787826267.695043:0:66551:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d9114ad00 x1874670011411968/t8589942965(8589942965) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826285 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:0.0:1787826267.695051:0:66551:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) for lock 0xe5f736940357b7cb 00000080:00010000:0.0:1787826267.695059:0:66551:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b7cb from it ffff9dd141cdbcc0 00010000:00010000:0.0:1787826267.695062:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a200/0xe5f736940357b7cb lrc: 3/0,1 mode: PW/PW res: [0x200000404:0x200:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0fbe8 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.695070:0:66551:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a200/0xe5f736940357b7cb lrc: 2/0,0 mode: PW/PW res: [0x200000404:0x200:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc0fbe8 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826267.695200:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826267.695214:0:66551:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826267.695217:0:66551:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826267.695334:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826267.695349:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:0.0:1787826267.695355:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000100:00100000:1.0:1787826267.695465:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011412096:192.168.203.104@tcp:1:lfs.0 00000100:00100000:1.0:1787826267.695899:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1787826267.695909:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011412224:192.168.203.104@tcp:1:lfs.0 00000100:00100000:3.0:1787826267.697992:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826267.698096:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025485312 00000100:00100000:2.0:1787826267.698107:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025485312 00000100:00100000:2.0:1787826267.698184:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7134 00000100:00100000:2.0:1787826267.698195:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d835270c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:9562:x1874670025485312:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826267.698217:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0fbbe expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826267.698322:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826267.698329:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d835270c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:9562:x1874670025485312:12345-192.168.203.104@tcp:104: Request processed in 137us (350us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826267.698339:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7134 00010000:00010000:0.0:1787826267.698675:0:40347:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826267.698684:0:40347:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0fbbe expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.698700:0:40347:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883bc00 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826267.698706:0:40347:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800428400000000 nid: local remote: 0x7918213f4fc0fbbe expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826267.698780:0:40347:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1787826267.698797:0:40347:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1787826267.698851:0:40347:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 2/0,0 mode: --/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0fbbe expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.698860:0:40347:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826267.698893:0:40347:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0fbbe expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.698903:0:40347:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000abd7405e/0xe5f736940357b7b6 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0fbbe expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.698929:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826267.698931:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826267.698933:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826267.698942:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1787826267.698959:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011412352:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1787826267.704027:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826267.704123:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025485440 00000100:00100000:3.0:1787826267.704284:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025485440 00000100:00100000:3.0:1787826267.704428:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7135 00000100:00100000:3.0:1787826267.704439:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91148f00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:28560:x1874670025485440:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826267.704463:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0fbc5 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826267.704698:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826267.704705:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91148f00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:28560:x1874670025485440:12345-192.168.203.104@tcp:104: Request processed in 269us (694us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826267.704714:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7135 00000100:00100000:1.0:1787826267.708031:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011412352:192.168.203.104@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1787826267.709050:0:61297:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826267.709066:0:61297:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0fbc5 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.709096:0:61297:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883b000 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826267.709104:0:61297:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800428400000000 nid: local remote: 0x7918213f4fc0fbc5 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826267.709129:0:61297:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1787826267.709154:0:61297:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1787826267.709361:0:61297:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 2/0,0 mode: --/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0fbc5 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826267.709370:0:61297:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826267.709405:0:61297:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 1/0,0 mode: --/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0fbc5 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826267.709414:0:61297:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b7bd lrc: 0/0,0 mode: --/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0fbc5 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826267.709443:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826267.709446:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826267.709448:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826267.709462:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1787826267.709483:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011412480:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826267.713787:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011412480:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826267.718280:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011412096:192.168.203.104@tcp:1:lfs.0 00000100:00100000:1.0:1787826267.718332:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011412224:192.168.203.104@tcp:1:lfs.0 00010000:00010000:0.0:1787826267.718525:0:66551:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:0.0:1787826267.718557:0:66551:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x200:0x0]: losing layout lock 00010000:00010000:0.0:1787826267.718586:0:66551:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1787826267.718595:0:66551:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x200:0x0]: losing layout lock 00010000:00010000:0.0:1787826267.718659:0:66551:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 1/0,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1787826267.718668:0:66551:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 1/0,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826267.718675:0:66551:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1787826267.718678:0:66551:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d86042c00/0xe5f736940357b7af lrc: 0/0,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0fbb7 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1787826267.718696:0:66551:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b770 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0fa83 expref: -99 pid: 66521 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000020:00100000:0.0:1787826267.718717:0:66551:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1787826267.718727:0:66551:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90845a40 x1874670011412736/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 472/440 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787826267.718949:0:66551:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011412736:192.168.203.104@tcp:36:lfs.0 00000100:00100000:0.0:1787826267.719243:0:66551:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd0c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826267.742925:0:66551:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589942966 gen 1 00000100:00100000:0.0:1787826267.742955:0:66551:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d908a3c00 x1874670011409664/t8589942963(8589942963) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826285 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1787826267.742968:0:66551:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d835261c0 x1874670011411584/t8589942964(8589942964) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826285 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1787826267.742977:0:66551:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d9114ad00 x1874670011411968/t8589942965(8589942965) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826285 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1787826267.742989:0:66551:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011412736:192.168.203.104@tcp:36:lfs.0 00010000:00010000:0.0:1787826267.743056:0:66551:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 31 res 8589935620/512 (0 0) 00000080:00010000:0.0:1787826267.743070:0:66551:0:(file.c:7076:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000404:0x200:0x0](ffff907d93313248) 00000002:00010000:0.0:1787826267.743085:0:66551:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x200:0x0]) in obj [0x200000404:0x200:0x0], intent: layout flags 0 00010000:00010000:0.0:1787826267.743158:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 66551 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.743170:0:66551:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66551 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.743181:0:66551:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66551 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826267.743206:0:66551:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011412864:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826267.743291:0:66551:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd0c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826267.751582:0:66551:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011412864:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826267.751672:0:66551:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.751690:0:66551:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826267.751708:0:66551:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x400 disposition=0x1, status=0 req@ffff907d90845a40 x1874670011412864/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/896 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826267.751720:0:66551:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1787826267.751735:0:66551:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.751750:0:66551:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/512 (0 0) 00000080:00200000:0.0:1787826267.751758:0:66551:0:(namei.c:952:ll_intent_lock()) intent lock 1024 on i1 [0x200000404:0x200:0x0] suppgids 0 0: rc 0 00000080:00010000:0.0:1787826267.751782:0:66551:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) for lock 0xe5f736940357b7d2 00000080:00010000:0.0:1787826267.751792:0:66551:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b7d2 from it ffff9dd141cdbc70 00010000:00010000:0.0:1787826267.751796:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.751806:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.751833:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.751846:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:0.0:1787826267.751856:0:66551:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x200:0x0](000000001c1f1433) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.751909:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.751918:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826267.751936:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826267.751949:0:66551:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826267.751953:0:66551:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826267.751961:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826267.751972:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:0.0:1787826267.751979:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:0.0:1787826267.752007:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1787826267.752019:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.752027:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1787826267.752040:0:66551:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x200:0x0](000000001c1f1433) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826267.752056:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826267.752073:0:66551:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826267.752075:0:66551:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826267.752077:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826267.752080:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:0.0:1787826267.752084:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:0.0:1787826267.752092:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1787826267.752294:0:66551:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x200:0x0]: layout version change: 4294967294 -> 7 00010000:00010000:0.0:1787826267.752300:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.752308:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200080:0.0:1787826267.752334:0:66551:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248) cmd=66f4 arg=ffffffffffffffff 00000080:00200008:0.0:1787826267.752350:0:66551:0:(file.c:6313:ll_getattr_dentry()) START file f204b.sanity-flr:[0x200000404:0x200:0x0](ffff907d93313248), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1787826267.752358:0:66551:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248),name=f204b.sanity-flr 00000002:00010000:0.0:1787826267.752374:0:66551:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x200:0x0]) in obj [0x200000404:0x200:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826267.752381:0:66551:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 30 res 8589935620/512 (0 0) 00010000:00010000:0.0:1787826267.752424:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 66551 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.752431:0:66551:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 66551 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.752445:0:66551:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 66551 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826267.752468:0:66551:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011412992:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826267.752586:0:66551:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd0c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826267.759727:0:66551:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011412992:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826267.759762:0:66551:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.759778:0:66551:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.759793:0:66551:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.759805:0:66551:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.759820:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.759828:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.759835:0:66551:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826267.759855:0:66551:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d90845a40 x1874670011412992/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/1136 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826267.759867:0:66551:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1787826267.759881:0:66551:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.759897:0:66551:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/512 (0 0) 00000080:00200000:0.0:1787826267.759906:0:66551:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x200:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1787826267.759987:0:66551:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x200:0x0]: layout version change: 7 -> 7 00000080:00010000:0.0:1787826267.759993:0:66551:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) for lock 0xe5f736940357b7d9 00000080:00010000:0.0:1787826267.759999:0:66551:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) 00000080:00010000:0.0:1787826267.760003:0:66551:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b7d9 from it ffff9dd141cdbcd8 00010000:00010000:0.0:1787826267.760006:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.760015:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826267.760054:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/1, layout version 6 restore needed 0 00020000:00200000:0.0:1787826267.760067:0:66551:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826267.760071:0:66551:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:0.0:1787826267.760075:0:66551:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x200:0x0] 00000008:00010000:0.0:1787826267.760103:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1787826267.760107:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d998fd0f0, osc lock ffff907d96368c30, flags 0x400000400001002 00000008:00010000:0.0:1787826267.760116:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1787826267.760117:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d998fd130, osc lock ffff907d963684e0, flags 0x400000400001002 00010000:00010000:0.0:1787826267.760240:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.760251:0:66551:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.760263:0:66551:0:(ldlm_request.c:1139:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff907d90845a40 x1874670011413120/t0(0) o101->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826267.760358:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011413120:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826267.760548:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.760558:0:66551:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [1048576->68157439] (req 1048576->68157439) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826267.760568:0:66551:0:(ldlm_request.c:1139:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff907d908461c0 x1874670011413248/t0(0) o101->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826267.764360:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1787826267.764390:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011413248:192.168.203.104@tcp:101:lfs.0 00000020:00200000:0.0:1787826267.765185:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00010000:00010000:1.0:1787826267.767156:0:2348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.767199:0:2348:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.767218:0:2348:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.767232:0:2348:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826267.767252:0:2348:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 1048576, setting rss=1048576; kms=1048576, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1787826267.767266:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826267.767271:0:2348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.767279:0:2348:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.767286:0:2348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.767295:0:2348:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1787826267.767327:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011413120:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826267.772580:0:2348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [1048576->68157439] (req 1048576->68157439) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.772663:0:2348:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.772684:0:2348:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.772698:0:2348:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826267.772717:0:2348:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1787826267.772732:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1787826267.772741:0:2348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.772750:0:2348:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.772758:0:2348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826267.772769:0:2348:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1787826267.772795:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011413248:192.168.203.104@tcp:101:lfs.0 00000080:00200000:0.0:1787826267.774306:0:66551:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x200:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:0.0:1787826267.774323:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_release():245 00000020:00010000:0.0:1787826267.774328:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1787826267.774334:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d998fd0f0 (ffff907d884a73f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1787826267.774341:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d998fd130 (ffff907d884a7908/1) at cl_lock_cancel():142 00000020:00010000:0.0:1787826267.774345:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1787826267.774348:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d998fd0f0 (ffff907d884a7f38/1) at cl_lock_fini():72 00000020:00010000:0.0:1787826267.774356:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d998fd130 (ffff907d884a7f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1787826267.774377:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:0.0:1787826267.919654:0:66551:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248) cmd=400866b1 arg=1 00000080:00200008:0.0:1787826267.919681:0:66551:0:(file.c:6313:ll_getattr_dentry()) START file f204b.sanity-flr:[0x200000404:0x200:0x0](ffff907d93313248), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1787826267.919693:0:66551:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248),name=f204b.sanity-flr 00000002:00010000:0.0:1787826267.919722:0:66551:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x200:0x0]) in obj [0x200000404:0x200:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826267.919749:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.919767:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826267.919777:0:66551:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x200:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826267.919787:0:66551:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) 00000080:00010000:0.0:1787826267.919791:0:66551:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) for lock 0xe5f736940357b7d9 00000080:00010000:0.0:1787826267.919800:0:66551:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b7d9 from it ffff9dd141cdbcd8 00010000:00010000:0.0:1787826267.919804:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.919815:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826267.919855:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:0.0:1787826267.919870:0:66551:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826267.919874:0:66551:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:0.0:1787826267.919878:0:66551:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x200:0x0] 00000008:00010000:0.0:1787826267.919903:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1787826267.919907:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d998fd0f0, osc lock ffff907d963684e0, flags 0x400000400001002 00000008:00010000:0.0:1787826267.919915:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1787826267.919916:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d998fd130, osc lock ffff907d96368c30, flags 0x400000400001002 00010000:00010000:0.0:1787826267.919938:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.919950:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:0.0:1787826267.919963:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1787826267.919967:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.919975:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.919991:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.920000:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 68157439) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:0.0:1787826267.920008:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1787826267.920011:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826267.920019:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:0.0:1787826267.920028:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1787826267.920040:0:66551:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x200:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:0.0:1787826267.920045:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_release():245 00000020:00010000:0.0:1787826267.920048:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1787826267.920051:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d998fd0f0 (ffff907d884a7908/1) at cl_lock_cancel():142 00000020:00010000:0.0:1787826267.920055:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d998fd130 (ffff907d884a73f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1787826267.920058:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1787826267.920062:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d998fd0f0 (ffff907d884a7f38/1) at cl_lock_fini():72 00000020:00010000:0.0:1787826267.920066:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d998fd130 (ffff907d884a7f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1787826267.920078:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1787826267.920099:0:66551:0:(file.c:5224:ll_file_seek()) START file f204b.sanity-flr:[0x200000404:0x200:0x0], offset: 0, type: SEEK_DATA 00000080:00200000:0.0:1787826267.920106:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:0.0:1787826267.920111:0:66551:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826267.920113:0:66551:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826267.920119:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826267.920124:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:0.0:1787826267.920127:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:0.0:1787826267.920145:0:66551:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x200:0x0] updating i_size 2097152 i_blocks 4096 00000080:00200000:0.0:1787826267.920155:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1787826267.920161:0:66551:0:(file.c:1848:ll_io_set_mirror()) f204b.sanity-flr: desiginated mirror: 1 00000080:00200000:0.0:1787826267.920163:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826267.920169:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826267.920172:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:0.0:1787826267.920175:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:0.0:1787826267.920178:0:66551:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 18446744073709551615] 00000008:00010000:0.0:1787826267.920183:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:0.0:1787826267.920186:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d998fd0f0, osc lock ffff907d96368c30, flags 0x0 00000008:00010000:0.0:1787826267.920188:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:0.0:1787826267.920189:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d998fd130, osc lock ffff907d963684e0, flags 0x0 00010000:00010000:0.0:1787826267.920199:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.920214:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.920223:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.920231:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.920238:0:66551:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.920247:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.920254:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 68157439) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.920263:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.920271:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826267.920284:0:66551:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826267.920293:0:66551:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x200:0x0] updating i_size 2097152 i_blocks 4096 00000100:00100000:0.0:1787826267.920418:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011413376:192.168.203.104@tcp:23:lfs.0 00000100:00100000:1.0:1787826267.921074:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011413504:192.168.203.104@tcp:23:lfs.0 00000100:00100000:1.0:1787826267.927786:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011413504:192.168.203.104@tcp:23:lfs.0 00000100:00100000:0.0:1787826267.928090:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011413376:192.168.203.104@tcp:23:lfs.0 00000020:00010000:2.0:1787826267.928170:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_release():245 00000020:00010000:2.0:1787826267.928175:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.928183:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d998fd0f0 (ffff907d884a7908/1) at cl_lock_cancel():142 00010000:00010000:2.0:1787826267.928197:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826267.928217:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00010000:2.0:1787826267.928239:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d998fd130 (ffff907d884a73f8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1787826267.928244:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826267.928253:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00010000:2.0:1787826267.928261:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826267.928265:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d998fd0f0 (ffff907d884a7f38/1) at cl_lock_fini():72 00000020:00010000:2.0:1787826267.928271:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d998fd130 (ffff907d884a7f38/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826267.928303:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826267.928315:0:66551:0:(file.c:5262:ll_file_seek()) COMPLETED file f204b.sanity-flr:[0x200000404:0x200:0x0], offset: 0, type: SEEK_DATA, rc = 0 00000080:00200008:2.0:1787826267.928335:0:66551:0:(file.c:5224:ll_file_seek()) START file f204b.sanity-flr:[0x200000404:0x200:0x0], offset: 0, type: SEEK_HOLE 00000080:00200000:2.0:1787826267.928346:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:2.0:1787826267.928357:0:66551:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826267.928360:0:66551:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1787826267.928366:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1787826267.928376:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1787826267.928384:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:2.0:1787826267.928418:0:66551:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x200:0x0] updating i_size 2097152 i_blocks 4096 00000080:00200000:2.0:1787826267.928427:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826267.928434:0:66551:0:(file.c:1848:ll_io_set_mirror()) f204b.sanity-flr: desiginated mirror: 1 00000080:00200000:2.0:1787826267.928436:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1787826267.928443:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1787826267.928447:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1787826267.928450:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:2.0:1787826267.928454:0:66551:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 18446744073709551615] 00000008:00010000:2.0:1787826267.928470:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:2.0:1787826267.928474:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d870bb4b0, osc lock ffff907d830a29c0, flags 0x0 00000008:00010000:2.0:1787826267.928479:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:2.0:1787826267.928481:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d870bb4f0, osc lock ffff907d830a3380, flags 0x0 00010000:00010000:2.0:1787826267.928515:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826267.928531:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826267.928546:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826267.928555:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826267.928563:0:66551:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826267.928577:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826267.928592:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 68157439) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826267.928601:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826267.928644:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826267.928652:0:66551:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1787826267.928665:0:66551:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x200:0x0] updating i_size 2097152 i_blocks 4096 00000100:00100000:3.0:1787826267.930340:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011413632:192.168.203.104@tcp:23:lfs.0 00000100:00100000:3.0:1787826267.930490:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1787826267.930498:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011413760:192.168.203.104@tcp:23:lfs.0 00000100:00100000:3.0:1787826267.934286:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011413760:192.168.203.104@tcp:23:lfs.0 00000100:00100000:3.0:1787826267.934327:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011413632:192.168.203.104@tcp:23:lfs.0 00000020:00010000:2.0:1787826267.934472:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_release():245 00000020:00010000:2.0:1787826267.934478:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.934485:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d870bb4b0 (ffff907d884a7908/1) at cl_lock_cancel():142 00010000:00010000:2.0:1787826267.934500:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826267.934535:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:2.0:1787826267.934556:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d870bb4f0 (ffff907d884a73f8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1787826267.934561:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826267.934570:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:2.0:1787826267.934578:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826267.934582:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d870bb4b0 (ffff907d884a7f38/1) at cl_lock_fini():72 00000020:00010000:2.0:1787826267.934588:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d870bb4f0 (ffff907d884a7f38/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826267.934662:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826267.934674:0:66551:0:(file.c:5262:ll_file_seek()) COMPLETED file f204b.sanity-flr:[0x200000404:0x200:0x0], offset: 0, type: SEEK_HOLE, rc = 2097152 00000080:00200080:2.0:1787826267.934692:0:66551:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248) cmd=400866b1 arg=0 00000080:00200080:2.0:1787826267.934705:0:66551:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248) cmd=400866b1 arg=1 00000080:00200008:2.0:1787826267.934713:0:66551:0:(file.c:6313:ll_getattr_dentry()) START file f204b.sanity-flr:[0x200000404:0x200:0x0](ffff907d93313248), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1787826267.934722:0:66551:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248),name=f204b.sanity-flr 00000002:00010000:2.0:1787826267.934747:0:66551:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x200:0x0]) in obj [0x200000404:0x200:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826267.934959:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826267.934977:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1787826267.934986:0:66551:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x200:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826267.934995:0:66551:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) 00000080:00010000:2.0:1787826267.934999:0:66551:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) for lock 0xe5f736940357b7d9 00000080:00010000:2.0:1787826267.935006:0:66551:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b7d9 from it ffff9dd141cdbcd8 00010000:00010000:2.0:1787826267.935009:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826267.935017:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1787826267.935173:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:2.0:1787826267.935190:0:66551:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826267.935194:0:66551:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1787826267.935199:0:66551:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x200:0x0] 00000008:00010000:2.0:1787826267.935224:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826267.935228:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d870bb4b0, osc lock ffff907d830a3380, flags 0x400000400001002 00000008:00010000:2.0:1787826267.935236:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826267.935237:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d870bb4f0, osc lock ffff907d830a29c0, flags 0x400000400001002 00010000:00010000:2.0:1787826267.935256:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826267.935267:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:2.0:1787826267.935280:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826267.935284:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826267.935292:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826267.935386:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826267.935401:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 68157439) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:2.0:1787826267.935411:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826267.936006:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826267.936019:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:2.0:1787826267.936032:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826267.936241:0:66551:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x200:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826267.936248:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_release():245 00000020:00010000:2.0:1787826267.936251:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.936255:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d870bb4b0 (ffff907d884a73f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.936260:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d870bb4f0 (ffff907d884a7908/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.936263:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826267.936266:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d870bb4b0 (ffff907d884a7f38/1) at cl_lock_fini():72 00000020:00010000:2.0:1787826267.936271:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d870bb4f0 (ffff907d884a7f38/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826267.936285:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826267.936403:0:66551:0:(file.c:2485:do_file_read_iter()) START file f204b.sanity-flr:[0x200000404:0x200:0x0], ppos: 0, count: 2097152 00000080:00200000:2.0:1787826267.936416:0:66551:0:(file.c:2090:ll_file_io_generic()) f204b.sanity-flr: read ppos: 0, bytes: 2097152 00000080:00200000:2.0:1787826267.936429:0:66551:0:(file.c:1848:ll_io_set_mirror()) f204b.sanity-flr: desiginated mirror: 1 00000020:00200000:2.0:1787826267.936436:0:66551:0:(cl_io.c:218:cl_io_rw_init()) header@000000008ae07a1f[0x110, 2, [0x200000404:0x200:0x0] hash] 00000020:00200000:2.0:1787826267.936439:0:66551:0:(cl_io.c:218:cl_io_rw_init()) io range: 1 [0, 2097152) 2048 0 00000080:00200000:2.0:1787826267.936444:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00000080:00200000:2.0:1787826267.936461:0:66551:0:(file.c:2170:ll_file_io_generic()) Range lock [0, 511] 00020000:00200000:2.0:1787826267.936470:0:66551:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 2097152, 1048576 00020000:00200000:2.0:1787826267.936474:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1787826267.936494:0:66551:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:2.0:1787826267.936518:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 11 00000008:00010000:2.0:1787826267.936523:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d963223b0, osc lock ffff907d830a29c0, flags 0x40000 00000080:00200000:2.0:1787826267.936530:0:66551:0:(vvp_io.c:874:vvp_io_read_start()) f204b.sanity-flr: read [0, 1048576) 00000080:00200000:2.0:1787826267.936537:0:66551:0:(vvp_io.c:194:vvp_prep_size()) [0x200000404:0x200:0x0] updating i_size 2097152 00000080:00200000:2.0:1787826267.936543:0:66551:0:(vvp_io.c:932:vvp_io_read_start()) count:1048576 iocb pos:0 00000080:00200000:2.0:1787826267.936551:0:66551:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248), size=1048576 (max 335544320), offset=0=0x0, pages 256 (max 81920), lockless, parallel 00000020:00200000:2.0:1787826267.937221:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1787826267.937226:0:66551:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 1048576 00000020:00010000:2.0:1787826267.937235:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_release():245 00000020:00010000:2.0:1787826267.937238:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.937241:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d963223b0 (ffff907d884a7908/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.937244:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826267.937246:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d963223b0 (ffff907d884a7f38/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826267.937255:0:66551:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00020000:00200000:2.0:1787826267.937261:0:66551:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 1048576 chunk: [1048576, 2097152) 2097152, 1048576 00020000:00200000:2.0:1787826267.937264:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00000008:00010000:2.0:1787826267.937364:0:65304:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907da56f8180 00000008:00100000:2.0:1787826267.937473:0:65304:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d911492c0 - object 0x240000400:804 offset 0<>1048576 00000100:00100000:3.0:1787826267.937571:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011413888:192.168.203.104@tcp:3:lfs.0 00000080:00200000:2.0:1787826267.941174:0:66551:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [256, 511] 00000008:00010000:2.0:1787826267.941208:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 11 00000008:00010000:2.0:1787826267.941214:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d963223b0, osc lock ffff907d830a29c0, flags 0x40000 00000080:00200000:2.0:1787826267.941227:0:66551:0:(vvp_io.c:874:vvp_io_read_start()) f204b.sanity-flr: read [1048576, 2097152) 00000080:00200000:2.0:1787826267.941240:0:66551:0:(vvp_io.c:194:vvp_prep_size()) [0x200000404:0x200:0x0] updating i_size 2097152 00000080:00200000:2.0:1787826267.941249:0:66551:0:(vvp_io.c:932:vvp_io_read_start()) count:1048576 iocb pos:1048576 00000080:00200000:2.0:1787826267.941257:0:66551:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248), size=1048576 (max 335544320), offset=1048576=0x100000, pages 256 (max 81920), lockless, parallel 00000020:00200000:2.0:1787826267.941732:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1787826267.941737:0:66551:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 2097152 00000020:00010000:2.0:1787826267.941748:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_release():245 00000020:00010000:2.0:1787826267.941752:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.941755:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d963223b0 (ffff907d884a73f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.941759:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826267.941762:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d963223b0 (ffff907d884a7f38/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826267.943330:0:66551:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1787826267.943351:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000008:00010000:2.0:1787826267.943383:0:65304:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907da56f8b40 00000008:00100000:2.0:1787826267.945038:0:65304:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d9114b840 - object 0x240000400:805 offset 1048576<>2097152 00000100:00100000:2.0:1787826267.945882:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011414016:192.168.203.104@tcp:3:lfs.0 00000008:00010000:2.0:1787826267.979364:0:2350:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907da56f8180 00000020:00200000:2.0:1787826267.979380:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1787826267.979385:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1787826267.979386:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1787826267.979388:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1787826267.979390:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1787826267.979391:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1787826267.979393:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1787826267.979395:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1787826267.979397:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1787826267.979464:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1787826267.979467:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1787826267.979468:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1787826267.979470:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1787826267.979472:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1787826267.979474:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1787826267.979476:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1787826267.979478:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1787826267.979480:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1787826267.979482:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1787826267.979484:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1787826267.979486:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1787826267.979488:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1787826267.979490:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1787826267.979492:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1787826267.979493:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1787826267.979495:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1787826267.979497:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1787826267.979499:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1787826267.979584:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1787826267.979588:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1787826267.979590:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1787826267.979592:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1787826267.979594:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1787826267.979596:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1787826267.979598:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1787826267.979599:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1787826267.979601:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1787826267.979603:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1787826267.979605:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1787826267.979626:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1787826267.979628:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1787826267.979630:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1787826267.979632:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1787826267.979633:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1787826267.979635:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1787826267.979637:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1787826267.979639:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1787826267.979640:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1787826267.979641:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1787826267.979643:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1787826267.979654:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1787826267.979656:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1787826267.979658:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1787826267.979661:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1787826267.979662:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1787826267.979663:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1787826267.979665:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1787826267.979667:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1787826267.979668:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1787826267.979670:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1787826267.979671:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1787826267.979673:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1787826267.979674:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1787826267.979676:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1787826267.979677:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1787826267.979680:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1787826267.979681:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1787826267.979683:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1787826267.979685:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1787826267.979686:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1787826267.979688:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1787826267.979690:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1787826267.979691:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1787826267.979693:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1787826267.979694:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1787826267.979696:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1787826267.979697:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1787826267.979700:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1787826267.979701:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1787826267.979703:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1787826267.979704:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1787826267.979706:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1787826267.979708:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1787826267.979710:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1787826267.979712:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1787826267.979714:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1787826267.979716:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1787826267.979718:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1787826267.979719:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1787826267.979721:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1787826267.979723:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1787826267.979724:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1787826267.979726:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1787826267.979728:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1787826267.979730:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1787826267.979732:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1787826267.979734:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1787826267.979735:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1787826267.979737:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1787826267.979739:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1787826267.979741:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1787826267.979743:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1787826267.979744:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1787826267.979746:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1787826267.979748:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1787826267.979750:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1787826267.979752:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1787826267.979753:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1787826267.979755:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1787826267.979757:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1787826267.979759:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1787826267.979760:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1787826267.979762:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1787826267.979763:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1787826267.979765:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1787826267.979767:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1787826267.979769:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1787826267.979771:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1787826267.979773:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1787826267.979774:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1787826267.979776:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1787826267.979777:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1787826267.979779:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1787826267.979781:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1787826267.979782:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1787826267.979784:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1787826267.979786:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1787826267.979788:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1787826267.979789:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1787826267.979791:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1787826267.979793:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1787826267.979795:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1787826267.979797:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1787826267.979798:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1787826267.979800:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1787826267.979801:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1787826267.979803:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1787826267.979805:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1787826267.979808:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1787826267.979810:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1787826267.979811:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1787826267.979813:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1787826267.979815:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1787826267.979817:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1787826267.979818:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1787826267.979820:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1787826267.979821:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1787826267.979824:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1787826267.979825:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1787826267.979827:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1787826267.979829:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1787826267.979830:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1787826267.979832:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1787826267.979834:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1787826267.979835:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1787826267.979836:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1787826267.979838:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1787826267.979839:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1787826267.979841:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1787826267.979842:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1787826267.979844:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1787826267.979846:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1787826267.979847:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1787826267.979849:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1787826267.979851:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1787826267.979853:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1787826267.979855:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1787826267.979857:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1787826267.979858:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1787826267.979860:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1787826267.979862:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1787826267.979863:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1787826267.979865:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1787826267.979867:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1787826267.979869:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1787826267.979870:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1787826267.980161:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1787826267.980164:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1787826267.980166:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1787826267.980168:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1787826267.980170:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1787826267.980171:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1787826267.980173:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1787826267.980174:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1787826267.980176:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1787826267.980177:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1787826267.980179:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1787826267.980181:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1787826267.980183:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1787826267.980185:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1787826267.980186:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1787826267.980189:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1787826267.980191:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1787826267.980193:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1787826267.980194:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1787826267.980196:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1787826267.980198:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1787826267.980200:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1787826267.980201:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1787826267.980203:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1787826267.980204:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1787826267.980205:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1787826267.980207:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1787826267.980209:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1787826267.980211:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1787826267.980213:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1787826267.980215:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1787826267.980217:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1787826267.980282:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1787826267.980286:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1787826267.980287:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1787826267.980289:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1787826267.980291:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1787826267.980293:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1787826267.980294:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1787826267.980296:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1787826267.980298:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1787826267.980300:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1787826267.980302:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1787826267.980304:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1787826267.980306:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1787826267.980309:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1787826267.980311:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1787826267.980312:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1787826267.980314:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1787826267.980316:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1787826267.980318:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1787826267.980320:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1787826267.980322:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1787826267.980323:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1787826267.980325:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1787826267.980327:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1787826267.980328:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1787826267.980330:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1787826267.980332:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1787826267.980334:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1787826267.980336:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1787826267.980337:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1787826267.980339:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1787826267.980341:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1787826267.980342:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1787826267.980344:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1787826267.980346:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1787826267.980348:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1787826267.980350:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1787826267.980351:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1787826267.980353:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1787826267.980354:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1787826267.980356:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1787826267.980357:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1787826267.980359:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1787826267.980361:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1787826267.980362:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1787826267.980364:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1787826267.980366:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1787826267.980368:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1787826267.981336:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1787826267.981351:0:2350:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907da56f8180 00000100:00100000:2.0:1787826267.981392:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011413888:192.168.203.104@tcp:3:lfs.0 00000008:00010000:2.0:1787826267.983573:0:2351:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907da56f8b40 00000020:00200000:2.0:1787826267.983589:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1787826267.983599:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1787826267.983601:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1787826267.983603:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1787826267.983605:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1787826267.983988:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1787826267.983990:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1787826267.983992:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1787826267.983994:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1787826267.983996:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1787826267.983998:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1787826267.983999:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1787826267.984001:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1787826267.984003:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1787826267.984006:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1787826267.984008:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1787826267.984010:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1787826267.984012:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1787826267.984014:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1787826267.984015:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1787826267.984017:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1787826267.984019:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1787826267.984021:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1787826267.984023:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1787826267.984024:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1787826267.984026:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1787826267.984028:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1787826267.984030:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1787826267.984032:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1787826267.984034:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1787826267.984035:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1787826267.984037:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1787826267.984038:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1787826267.984040:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1787826267.984041:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1787826267.984168:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1787826267.984172:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1787826267.984174:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1787826267.984176:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1787826267.984177:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1787826267.984179:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1787826267.984181:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1787826267.984182:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1787826267.984184:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1787826267.984186:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1787826267.984188:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1787826267.984190:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1787826267.984192:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1787826267.984193:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1787826267.984195:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1787826267.984197:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1787826267.984198:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1787826267.984200:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1787826267.984201:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1787826267.984203:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1787826267.984204:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1787826267.984206:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1787826267.984207:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1787826267.984209:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1787826267.984210:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1787826267.984211:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1787826267.984213:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1787826267.984215:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1787826267.984216:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1787826267.984217:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1787826267.984219:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1787826267.984220:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1787826267.984222:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1787826267.984223:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1787826267.984225:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1787826267.984227:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1787826267.984228:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1787826267.984230:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1787826267.984231:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1787826267.984233:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1787826267.984234:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1787826267.984235:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1787826267.984236:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1787826267.984238:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1787826267.984240:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1787826267.984241:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1787826267.984243:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1787826267.984245:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1787826267.984246:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1787826267.984248:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1787826267.984249:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1787826267.984251:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1787826267.984252:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1787826267.984254:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1787826267.984255:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1787826267.984257:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1787826267.984259:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1787826267.984261:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1787826267.984263:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1787826267.984265:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1787826267.984267:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1787826267.984269:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1787826267.984270:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1787826267.984272:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1787826267.984274:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1787826267.984275:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1787826267.984277:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1787826267.984279:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1787826267.984281:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1787826267.984282:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1787826267.984284:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1787826267.984285:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1787826267.984287:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1787826267.984289:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1787826267.984290:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1787826267.984292:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1787826267.984294:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1787826267.984361:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1787826267.984363:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1787826267.984365:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1787826267.984368:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1787826267.984369:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1787826267.984371:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1787826267.984373:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1787826267.984375:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1787826267.984377:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1787826267.984379:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1787826267.984381:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1787826267.984382:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1787826267.984384:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1787826267.984386:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1787826267.984388:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1787826267.984389:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1787826267.984391:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1787826267.984393:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1787826267.984395:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1787826267.984397:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1787826267.984398:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1787826267.984400:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1787826267.984402:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1787826267.984403:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1787826267.984405:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1787826267.984407:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1787826267.984409:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1787826267.984410:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1787826267.984412:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1787826267.984414:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1787826267.984416:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1787826267.984418:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1787826267.984420:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1787826267.984422:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1787826267.984424:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1787826267.984426:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1787826267.984427:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1787826267.984429:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1787826267.984431:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1787826267.984432:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1787826267.984434:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1787826267.984436:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1787826267.984438:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1787826267.984440:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1787826267.984442:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1787826267.984444:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1787826267.984445:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1787826267.984447:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1787826267.984449:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1787826267.984451:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1787826267.984453:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1787826267.984454:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1787826267.984456:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1787826267.984457:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1787826267.984459:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1787826267.984461:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1787826267.984462:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1787826267.984464:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1787826267.984466:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1787826267.984468:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1787826267.984470:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1787826267.984472:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1787826267.984474:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1787826267.984476:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1787826267.984478:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1787826267.984480:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1787826267.984482:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1787826267.984483:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1787826267.984485:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1787826267.984487:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1787826267.984489:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1787826267.984491:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1787826267.984493:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1787826267.984494:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1787826267.984496:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1787826267.984498:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1787826267.984500:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1787826267.984502:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1787826267.984504:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1787826267.984517:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1787826267.984519:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1787826267.984521:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1787826267.984523:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1787826267.984524:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1787826267.984526:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1787826267.984527:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1787826267.984529:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1787826267.984531:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1787826267.984532:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1787826267.984534:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1787826267.984535:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1787826267.984537:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1787826267.984539:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1787826267.984541:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1787826267.984543:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1787826267.984545:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1787826267.984547:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1787826267.984549:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1787826267.984550:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1787826267.984552:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1787826267.984553:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1787826267.984555:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1787826267.984557:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1787826267.984559:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1787826267.984561:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1787826267.984563:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1787826267.984565:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1787826267.984566:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1787826267.984568:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1787826267.984570:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1787826267.984572:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1787826267.984574:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1787826267.984576:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1787826267.984578:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1787826267.984580:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1787826267.984582:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1787826267.984584:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1787826267.984586:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1787826267.984587:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1787826267.984589:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1787826267.984591:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1787826267.984593:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1787826267.984594:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1787826267.984596:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1787826267.984597:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1787826267.984599:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1787826267.984601:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1787826267.984603:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1787826267.984604:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1787826267.984632:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1787826267.984635:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1787826267.984637:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1787826267.984639:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1787826267.984640:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1787826267.984642:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1787826267.984644:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1787826267.984646:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1787826267.984648:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1787826267.984650:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1787826267.984652:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1787826267.984654:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1787826267.984656:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1787826267.984658:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1787826267.984659:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1787826267.984829:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1787826267.984935:0:2351:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907da56f8b40 00000100:00100000:2.0:1787826267.984969:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011414016:192.168.203.104@tcp:3:lfs.0 00000080:00200000:3.0:1787826267.985213:0:66551:0:(file.c:2207:ll_file_io_generic()) Range unlock [0, 511] 00000080:00200000:3.0:1787826267.985246:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787826267.985254:0:66551:0:(file.c:2240:ll_file_io_generic()) f204b.sanity-flr: 1 io complete with rc: 0, result: 2097152, restart: 0 00000020:00200000:3.0:1787826267.985258:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1787826267.985267:0:66551:0:(file.c:2326:ll_file_io_generic()) iot: 1, result: 2097152 00000080:00200080:3.0:1787826267.985292:0:66551:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248) cmd=400866b1 arg=0 00000080:00200080:3.0:1787826267.985307:0:66551:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248) cmd=400866b1 arg=2 00000080:00200008:3.0:1787826267.985316:0:66551:0:(file.c:6313:ll_getattr_dentry()) START file f204b.sanity-flr:[0x200000404:0x200:0x0](ffff907d93313248), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787826267.985325:0:66551:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248),name=f204b.sanity-flr 00000002:00010000:3.0:1787826267.985354:0:66551:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x200:0x0]) in obj [0x200000404:0x200:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826267.985380:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826267.985443:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1787826267.985456:0:66551:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x200:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826267.985466:0:66551:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) 00000080:00010000:3.0:1787826267.985469:0:66551:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) for lock 0xe5f736940357b7d9 00000080:00010000:3.0:1787826267.985480:0:66551:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b7d9 from it ffff9dd141cdbcd8 00010000:00010000:3.0:1787826267.985484:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826267.985495:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1787826267.985541:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:3.0:1787826267.985558:0:66551:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:3.0:1787826267.985561:0:66551:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:3.0:1787826267.985567:0:66551:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x200:0x0] 00000008:00010000:3.0:1787826267.985590:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826267.985595:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c7d330, osc lock ffff907d963b9c08, flags 0x400000400001002 00000008:00010000:3.0:1787826267.985601:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826267.985602:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c7d370, osc lock ffff907d963b9860, flags 0x400000400001002 00010000:00010000:3.0:1787826267.985929:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826267.986047:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:2.0:1787826267.986062:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826267.986066:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826267.986076:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1787826267.986103:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826267.986111:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 68157439) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:2.0:1787826267.986118:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826267.986121:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826267.986129:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00200000:2.0:1787826267.986136:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826267.986147:0:66551:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x200:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826267.986152:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_release():245 00000020:00010000:2.0:1787826267.986156:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.986158:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c7d330 (ffff907d884a73f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.986162:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c7d370 (ffff907d884a7908/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.986165:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826267.986168:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c7d330 (ffff907d884a7f38/1) at cl_lock_fini():72 00000020:00010000:2.0:1787826267.986173:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c7d370 (ffff907d884a7f38/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826267.986183:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826267.986216:0:66551:0:(file.c:2652:do_file_write_iter()) START file f204b.sanity-flr:[0x200000404:0x200:0x0], ppos: 0, count: 2097152 00000080:00200000:2.0:1787826267.986235:0:66551:0:(file.c:2090:ll_file_io_generic()) f204b.sanity-flr: write ppos: 0, bytes: 2097152 00000080:00200000:2.0:1787826267.986246:0:66551:0:(file.c:1848:ll_io_set_mirror()) f204b.sanity-flr: desiginated mirror: 2 00000020:00200000:2.0:1787826267.986253:0:66551:0:(cl_io.c:218:cl_io_rw_init()) header@000000008ae07a1f[0x110, 2, [0x200000404:0x200:0x0] hash] 00000020:00200000:2.0:1787826267.986256:0:66551:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [0, 2097152) 2048 0 00000080:00200000:2.0:1787826267.986259:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00000080:00200000:2.0:1787826267.986272:0:66551:0:(file.c:2170:ll_file_io_generic()) Range lock [0, 511] 00020000:00200000:2.0:1787826267.986281:0:66551:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 2097152) 2097152, 2097152 00020000:00200000:2.0:1787826267.986286:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[3] flags 0x11 00000080:00200000:2.0:1787826267.986305:0:66551:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 511] 00000008:00010000:2.0:1787826267.986317:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 11 00000008:00010000:2.0:1787826267.986322:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d963223b0, osc lock ffff907d830a29c0, flags 0x40000 00000080:00200000:2.0:1787826267.986331:0:66551:0:(vvp_io.c:1339:vvp_io_write_start()) f204b.sanity-flr: write [0, 2097152) append 0 append lockpos 0 00000080:00200000:2.0:1787826267.986362:0:66551:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248), size=2097152 (max 335544320), offset=0=0x0, pages 512 (max 81920), lockless, parallel 00000020:00200000:2.0:1787826267.986959:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 512 items remaining. 00000080:00200000:2.0:1787826267.986964:0:66551:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 2097152 count 2097152 file_offset 2097152 00000080:00200000:2.0:1787826267.986976:0:66551:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x200:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826267.986982:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_release():245 00000020:00010000:2.0:1787826267.986984:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.986987:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d963223b0 (ffff907d884a7908/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826267.986990:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826267.986993:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d963223b0 (ffff907d884a7f38/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826267.987000:0:66551:0:(vvp_io.c:533:vvp_io_advance()) advancing 2097152 bytes 00000020:00200000:2.0:1787826267.987007:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1787826267.987028:0:65304:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907da56f8b40 00000008:00100000:2.0:1787826267.988216:0:65304:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d9114b840 - object 0x280000400:777 offset 0<>2097152 00000100:00100000:3.0:1787826267.988371:0:2351:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1787826267.988467:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011414144:192.168.203.104@tcp:4:lfs.0 00010000:00010000:1.0:1787826268.021959:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826268.043540:0:2351:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0001-osc-ffff907d986d0000: committing for last_committed 17179873830 gen 1 00000008:00010000:3.0:1787826268.043699:0:2351:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907da56f8b40 00000020:00200000:3.0:1787826268.043709:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 511 items remaining. 00000020:00200000:3.0:1787826268.043712:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 510 items remaining. 00000020:00200000:3.0:1787826268.043714:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 509 items remaining. 00000020:00200000:3.0:1787826268.043717:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 508 items remaining. 00000020:00200000:3.0:1787826268.043718:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 507 items remaining. 00000020:00200000:3.0:1787826268.043720:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 506 items remaining. 00000020:00200000:3.0:1787826268.043722:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 505 items remaining. 00000020:00200000:3.0:1787826268.043724:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 504 items remaining. 00000020:00200000:3.0:1787826268.043726:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 503 items remaining. 00000020:00200000:3.0:1787826268.043727:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 502 items remaining. 00000020:00200000:3.0:1787826268.043729:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 501 items remaining. 00000020:00200000:3.0:1787826268.043730:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 500 items remaining. 00000020:00200000:3.0:1787826268.043732:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 499 items remaining. 00000020:00200000:3.0:1787826268.043734:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 498 items remaining. 00000020:00200000:3.0:1787826268.043735:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 497 items remaining. 00000020:00200000:3.0:1787826268.043736:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 496 items remaining. 00000020:00200000:3.0:1787826268.043737:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 495 items remaining. 00000020:00200000:3.0:1787826268.043739:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 494 items remaining. 00000020:00200000:3.0:1787826268.043741:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 493 items remaining. 00000020:00200000:3.0:1787826268.043743:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 492 items remaining. 00000020:00200000:3.0:1787826268.043745:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 491 items remaining. 00000020:00200000:3.0:1787826268.043747:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 490 items remaining. 00000020:00200000:3.0:1787826268.043749:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 489 items remaining. 00000020:00200000:3.0:1787826268.043751:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 488 items remaining. 00000020:00200000:3.0:1787826268.043752:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 487 items remaining. 00000020:00200000:3.0:1787826268.043754:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 486 items remaining. 00000020:00200000:3.0:1787826268.043756:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 485 items remaining. 00000020:00200000:3.0:1787826268.043758:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 484 items remaining. 00000020:00200000:3.0:1787826268.043759:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 483 items remaining. 00000020:00200000:3.0:1787826268.043761:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 482 items remaining. 00000020:00200000:3.0:1787826268.043763:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 481 items remaining. 00000020:00200000:3.0:1787826268.043765:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 480 items remaining. 00000020:00200000:3.0:1787826268.043767:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 479 items remaining. 00000020:00200000:3.0:1787826268.043768:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 478 items remaining. 00000020:00200000:3.0:1787826268.043770:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 477 items remaining. 00000020:00200000:3.0:1787826268.043772:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 476 items remaining. 00000020:00200000:3.0:1787826268.043774:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 475 items remaining. 00000020:00200000:3.0:1787826268.043775:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 474 items remaining. 00000020:00200000:3.0:1787826268.043777:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 473 items remaining. 00000020:00200000:3.0:1787826268.043779:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 472 items remaining. 00000020:00200000:3.0:1787826268.043781:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 471 items remaining. 00000020:00200000:3.0:1787826268.043783:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 470 items remaining. 00000020:00200000:3.0:1787826268.043785:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 469 items remaining. 00000020:00200000:3.0:1787826268.043787:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 468 items remaining. 00000020:00200000:3.0:1787826268.043789:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 467 items remaining. 00000020:00200000:3.0:1787826268.043791:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 466 items remaining. 00000020:00200000:3.0:1787826268.043792:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 465 items remaining. 00000020:00200000:3.0:1787826268.043794:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 464 items remaining. 00000020:00200000:3.0:1787826268.043796:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 463 items remaining. 00000020:00200000:3.0:1787826268.043797:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 462 items remaining. 00000020:00200000:3.0:1787826268.043799:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 461 items remaining. 00000020:00200000:3.0:1787826268.043801:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 460 items remaining. 00000020:00200000:3.0:1787826268.043803:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 459 items remaining. 00000020:00200000:3.0:1787826268.043804:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 458 items remaining. 00000020:00200000:3.0:1787826268.043806:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 457 items remaining. 00000020:00200000:3.0:1787826268.043808:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 456 items remaining. 00000020:00200000:3.0:1787826268.043810:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 455 items remaining. 00000020:00200000:3.0:1787826268.043812:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 454 items remaining. 00000020:00200000:3.0:1787826268.043813:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 453 items remaining. 00000020:00200000:3.0:1787826268.043815:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 452 items remaining. 00000020:00200000:3.0:1787826268.043817:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 451 items remaining. 00000020:00200000:3.0:1787826268.043818:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 450 items remaining. 00000020:00200000:3.0:1787826268.043820:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 449 items remaining. 00000020:00200000:3.0:1787826268.043822:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 448 items remaining. 00000020:00200000:3.0:1787826268.043823:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 447 items remaining. 00000020:00200000:3.0:1787826268.043825:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 446 items remaining. 00000020:00200000:3.0:1787826268.043827:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 445 items remaining. 00000020:00200000:3.0:1787826268.043829:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 444 items remaining. 00000020:00200000:3.0:1787826268.043831:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 443 items remaining. 00000020:00200000:3.0:1787826268.043833:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 442 items remaining. 00000020:00200000:3.0:1787826268.043835:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 441 items remaining. 00000020:00200000:3.0:1787826268.043837:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 440 items remaining. 00000020:00200000:3.0:1787826268.043838:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 439 items remaining. 00000020:00200000:3.0:1787826268.043840:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 438 items remaining. 00000020:00200000:3.0:1787826268.043842:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 437 items remaining. 00000020:00200000:3.0:1787826268.043843:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 436 items remaining. 00000020:00200000:3.0:1787826268.043845:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 435 items remaining. 00000020:00200000:3.0:1787826268.043847:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 434 items remaining. 00000020:00200000:3.0:1787826268.043848:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 433 items remaining. 00000020:00200000:3.0:1787826268.043850:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 432 items remaining. 00000020:00200000:3.0:1787826268.043851:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 431 items remaining. 00000020:00200000:3.0:1787826268.043853:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 430 items remaining. 00000020:00200000:3.0:1787826268.043854:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 429 items remaining. 00000020:00200000:3.0:1787826268.043856:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 428 items remaining. 00000020:00200000:3.0:1787826268.043858:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 427 items remaining. 00000020:00200000:3.0:1787826268.043860:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 426 items remaining. 00000020:00200000:3.0:1787826268.043861:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 425 items remaining. 00000020:00200000:3.0:1787826268.043863:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 424 items remaining. 00000020:00200000:3.0:1787826268.043864:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 423 items remaining. 00000020:00200000:3.0:1787826268.043866:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 422 items remaining. 00000020:00200000:3.0:1787826268.043867:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 421 items remaining. 00000020:00200000:3.0:1787826268.043869:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 420 items remaining. 00000020:00200000:3.0:1787826268.043870:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 419 items remaining. 00000020:00200000:3.0:1787826268.043872:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 418 items remaining. 00000020:00200000:3.0:1787826268.043874:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 417 items remaining. 00000020:00200000:3.0:1787826268.043876:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 416 items remaining. 00000020:00200000:3.0:1787826268.043877:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 415 items remaining. 00000020:00200000:3.0:1787826268.043879:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 414 items remaining. 00000020:00200000:3.0:1787826268.043880:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 413 items remaining. 00000020:00200000:3.0:1787826268.043882:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 412 items remaining. 00000020:00200000:3.0:1787826268.043884:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 411 items remaining. 00000020:00200000:3.0:1787826268.043886:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 410 items remaining. 00000020:00200000:3.0:1787826268.043887:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 409 items remaining. 00000020:00200000:3.0:1787826268.043889:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 408 items remaining. 00000020:00200000:3.0:1787826268.043891:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 407 items remaining. 00000020:00200000:3.0:1787826268.043892:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 406 items remaining. 00000020:00200000:3.0:1787826268.043894:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 405 items remaining. 00000020:00200000:3.0:1787826268.043896:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 404 items remaining. 00000020:00200000:3.0:1787826268.043897:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 403 items remaining. 00000020:00200000:3.0:1787826268.043899:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 402 items remaining. 00000020:00200000:3.0:1787826268.043900:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 401 items remaining. 00000020:00200000:3.0:1787826268.043902:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 400 items remaining. 00000020:00200000:3.0:1787826268.043904:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 399 items remaining. 00000020:00200000:3.0:1787826268.043905:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 398 items remaining. 00000020:00200000:3.0:1787826268.043907:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 397 items remaining. 00000020:00200000:3.0:1787826268.043909:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 396 items remaining. 00000020:00200000:3.0:1787826268.043910:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 395 items remaining. 00000020:00200000:3.0:1787826268.043912:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 394 items remaining. 00000020:00200000:3.0:1787826268.043914:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 393 items remaining. 00000020:00200000:3.0:1787826268.043915:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 392 items remaining. 00000020:00200000:3.0:1787826268.043917:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 391 items remaining. 00000020:00200000:3.0:1787826268.043919:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 390 items remaining. 00000020:00200000:3.0:1787826268.043921:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 389 items remaining. 00000020:00200000:3.0:1787826268.043923:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 388 items remaining. 00000020:00200000:3.0:1787826268.043924:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 387 items remaining. 00000020:00200000:3.0:1787826268.043926:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 386 items remaining. 00000020:00200000:3.0:1787826268.043928:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 385 items remaining. 00000020:00200000:3.0:1787826268.043930:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 384 items remaining. 00000020:00200000:3.0:1787826268.043932:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 383 items remaining. 00000020:00200000:3.0:1787826268.043934:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 382 items remaining. 00000020:00200000:3.0:1787826268.043935:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 381 items remaining. 00000020:00200000:3.0:1787826268.043937:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 380 items remaining. 00000020:00200000:3.0:1787826268.043939:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 379 items remaining. 00000020:00200000:3.0:1787826268.043941:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 378 items remaining. 00000020:00200000:3.0:1787826268.043942:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 377 items remaining. 00000020:00200000:3.0:1787826268.043944:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 376 items remaining. 00000020:00200000:3.0:1787826268.043946:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 375 items remaining. 00000020:00200000:3.0:1787826268.043948:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 374 items remaining. 00000020:00200000:3.0:1787826268.043950:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 373 items remaining. 00000020:00200000:3.0:1787826268.043953:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 372 items remaining. 00000020:00200000:3.0:1787826268.043954:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 371 items remaining. 00000020:00200000:3.0:1787826268.043956:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 370 items remaining. 00000020:00200000:3.0:1787826268.043958:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 369 items remaining. 00000020:00200000:3.0:1787826268.043960:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 368 items remaining. 00000020:00200000:3.0:1787826268.043962:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 367 items remaining. 00000020:00200000:3.0:1787826268.043964:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 366 items remaining. 00000020:00200000:3.0:1787826268.043965:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 365 items remaining. 00000020:00200000:3.0:1787826268.043967:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 364 items remaining. 00000020:00200000:3.0:1787826268.043969:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 363 items remaining. 00000020:00200000:3.0:1787826268.043971:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 362 items remaining. 00000020:00200000:3.0:1787826268.043972:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 361 items remaining. 00000020:00200000:3.0:1787826268.043974:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 360 items remaining. 00000020:00200000:3.0:1787826268.043976:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 359 items remaining. 00000020:00200000:3.0:1787826268.043978:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 358 items remaining. 00000020:00200000:3.0:1787826268.043979:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 357 items remaining. 00000020:00200000:3.0:1787826268.043981:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 356 items remaining. 00000020:00200000:3.0:1787826268.043982:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 355 items remaining. 00000020:00200000:3.0:1787826268.043984:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 354 items remaining. 00000020:00200000:3.0:1787826268.043985:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 353 items remaining. 00000020:00200000:3.0:1787826268.043987:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 352 items remaining. 00000020:00200000:3.0:1787826268.043989:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 351 items remaining. 00000020:00200000:3.0:1787826268.043990:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 350 items remaining. 00000020:00200000:3.0:1787826268.043992:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 349 items remaining. 00000020:00200000:3.0:1787826268.043993:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 348 items remaining. 00000020:00200000:3.0:1787826268.043995:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 347 items remaining. 00000020:00200000:3.0:1787826268.043997:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 346 items remaining. 00000020:00200000:3.0:1787826268.043998:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 345 items remaining. 00000020:00200000:3.0:1787826268.044000:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 344 items remaining. 00000020:00200000:3.0:1787826268.044002:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 343 items remaining. 00000020:00200000:3.0:1787826268.044004:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 342 items remaining. 00000020:00200000:3.0:1787826268.044006:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 341 items remaining. 00000020:00200000:3.0:1787826268.044008:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 340 items remaining. 00000020:00200000:3.0:1787826268.044009:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 339 items remaining. 00000020:00200000:3.0:1787826268.044011:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 338 items remaining. 00000020:00200000:3.0:1787826268.044013:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 337 items remaining. 00000020:00200000:3.0:1787826268.044015:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 336 items remaining. 00000020:00200000:3.0:1787826268.044016:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 335 items remaining. 00000020:00200000:3.0:1787826268.044018:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 334 items remaining. 00000020:00200000:3.0:1787826268.044019:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 333 items remaining. 00000020:00200000:3.0:1787826268.044021:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 332 items remaining. 00000020:00200000:3.0:1787826268.044023:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 331 items remaining. 00000020:00200000:3.0:1787826268.044025:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 330 items remaining. 00000020:00200000:3.0:1787826268.044026:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 329 items remaining. 00000020:00200000:3.0:1787826268.044028:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 328 items remaining. 00000020:00200000:3.0:1787826268.044029:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 327 items remaining. 00000020:00200000:3.0:1787826268.044031:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 326 items remaining. 00000020:00200000:3.0:1787826268.044032:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 325 items remaining. 00000020:00200000:3.0:1787826268.044034:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 324 items remaining. 00000020:00200000:3.0:1787826268.044036:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 323 items remaining. 00000020:00200000:3.0:1787826268.044037:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 322 items remaining. 00000020:00200000:3.0:1787826268.044039:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 321 items remaining. 00000020:00200000:3.0:1787826268.044041:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 320 items remaining. 00000020:00200000:3.0:1787826268.044043:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 319 items remaining. 00000020:00200000:3.0:1787826268.044044:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 318 items remaining. 00000020:00200000:3.0:1787826268.044167:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 317 items remaining. 00000020:00200000:3.0:1787826268.044170:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 316 items remaining. 00000020:00200000:3.0:1787826268.044171:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 315 items remaining. 00000020:00200000:3.0:1787826268.044173:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 314 items remaining. 00000020:00200000:3.0:1787826268.044175:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 313 items remaining. 00000020:00200000:3.0:1787826268.044177:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 312 items remaining. 00000020:00200000:3.0:1787826268.044178:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 311 items remaining. 00000020:00200000:3.0:1787826268.044180:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 310 items remaining. 00000020:00200000:3.0:1787826268.044182:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 309 items remaining. 00000020:00200000:3.0:1787826268.044184:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 308 items remaining. 00000020:00200000:3.0:1787826268.044185:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 307 items remaining. 00000020:00200000:3.0:1787826268.044188:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 306 items remaining. 00000020:00200000:3.0:1787826268.044189:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 305 items remaining. 00000020:00200000:3.0:1787826268.044191:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 304 items remaining. 00000020:00200000:3.0:1787826268.044193:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 303 items remaining. 00000020:00200000:3.0:1787826268.044195:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 302 items remaining. 00000020:00200000:3.0:1787826268.044196:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 301 items remaining. 00000020:00200000:3.0:1787826268.044198:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 300 items remaining. 00000020:00200000:3.0:1787826268.044200:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 299 items remaining. 00000020:00200000:3.0:1787826268.044202:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 298 items remaining. 00000020:00200000:3.0:1787826268.044204:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 297 items remaining. 00000020:00200000:3.0:1787826268.044205:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 296 items remaining. 00000020:00200000:3.0:1787826268.044207:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 295 items remaining. 00000020:00200000:3.0:1787826268.044209:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 294 items remaining. 00000020:00200000:3.0:1787826268.044210:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 293 items remaining. 00000020:00200000:3.0:1787826268.044212:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 292 items remaining. 00000020:00200000:3.0:1787826268.044213:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 291 items remaining. 00000020:00200000:3.0:1787826268.044215:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 290 items remaining. 00000020:00200000:3.0:1787826268.044217:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 289 items remaining. 00000020:00200000:3.0:1787826268.044218:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 288 items remaining. 00000020:00200000:3.0:1787826268.044220:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 287 items remaining. 00000020:00200000:3.0:1787826268.044222:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 286 items remaining. 00000020:00200000:3.0:1787826268.044224:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 285 items remaining. 00000020:00200000:3.0:1787826268.044225:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 284 items remaining. 00000020:00200000:3.0:1787826268.044227:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 283 items remaining. 00000020:00200000:3.0:1787826268.044229:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 282 items remaining. 00000020:00200000:3.0:1787826268.044230:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 281 items remaining. 00000020:00200000:3.0:1787826268.044232:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 280 items remaining. 00000020:00200000:3.0:1787826268.044234:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 279 items remaining. 00000020:00200000:3.0:1787826268.044236:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 278 items remaining. 00000020:00200000:3.0:1787826268.044238:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 277 items remaining. 00000020:00200000:3.0:1787826268.044240:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 276 items remaining. 00000020:00200000:3.0:1787826268.044241:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 275 items remaining. 00000020:00200000:3.0:1787826268.044244:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 274 items remaining. 00000020:00200000:3.0:1787826268.044246:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 273 items remaining. 00000020:00200000:3.0:1787826268.044248:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 272 items remaining. 00000020:00200000:3.0:1787826268.044250:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 271 items remaining. 00000020:00200000:3.0:1787826268.044251:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 270 items remaining. 00000020:00200000:3.0:1787826268.044253:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 269 items remaining. 00000020:00200000:3.0:1787826268.044256:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 268 items remaining. 00000020:00200000:3.0:1787826268.044258:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 267 items remaining. 00000020:00200000:3.0:1787826268.044260:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 266 items remaining. 00000020:00200000:3.0:1787826268.044262:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 265 items remaining. 00000020:00200000:3.0:1787826268.044264:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 264 items remaining. 00000020:00200000:3.0:1787826268.044266:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 263 items remaining. 00000020:00200000:3.0:1787826268.044268:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 262 items remaining. 00000020:00200000:3.0:1787826268.044270:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 261 items remaining. 00000020:00200000:3.0:1787826268.044272:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 260 items remaining. 00000020:00200000:3.0:1787826268.044274:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 259 items remaining. 00000020:00200000:3.0:1787826268.044276:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 258 items remaining. 00000020:00200000:3.0:1787826268.044278:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 257 items remaining. 00000020:00200000:3.0:1787826268.044280:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000020:00200000:3.0:1787826268.044282:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:3.0:1787826268.044283:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:3.0:1787826268.044285:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:3.0:1787826268.044288:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:3.0:1787826268.044289:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:3.0:1787826268.044291:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:3.0:1787826268.044293:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:3.0:1787826268.044295:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:3.0:1787826268.045718:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:3.0:1787826268.045721:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:3.0:1787826268.045723:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:3.0:1787826268.045725:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:3.0:1787826268.045727:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:3.0:1787826268.045729:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:3.0:1787826268.045731:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:3.0:1787826268.045733:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:3.0:1787826268.045735:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:3.0:1787826268.045737:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:3.0:1787826268.045739:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:3.0:1787826268.045742:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:3.0:1787826268.045745:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:3.0:1787826268.045746:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:3.0:1787826268.045749:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:3.0:1787826268.045750:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:3.0:1787826268.045752:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:3.0:1787826268.045754:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:3.0:1787826268.045756:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:3.0:1787826268.045758:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:3.0:1787826268.045760:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:3.0:1787826268.045761:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:3.0:1787826268.045763:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:3.0:1787826268.045766:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:3.0:1787826268.045768:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:3.0:1787826268.045769:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:3.0:1787826268.045772:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:3.0:1787826268.045774:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:3.0:1787826268.045776:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:3.0:1787826268.045777:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:3.0:1787826268.045779:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:3.0:1787826268.045781:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:3.0:1787826268.045782:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:3.0:1787826268.045784:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:3.0:1787826268.045786:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:3.0:1787826268.045788:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:3.0:1787826268.045789:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:3.0:1787826268.045791:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:3.0:1787826268.045793:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:3.0:1787826268.045795:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:3.0:1787826268.045796:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:3.0:1787826268.045799:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:3.0:1787826268.045800:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:3.0:1787826268.045803:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:3.0:1787826268.045805:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:3.0:1787826268.045806:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:3.0:1787826268.045808:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:3.0:1787826268.045810:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:3.0:1787826268.045812:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:3.0:1787826268.045814:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:3.0:1787826268.045816:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:3.0:1787826268.045817:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:3.0:1787826268.045819:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:3.0:1787826268.045821:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:3.0:1787826268.045823:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:3.0:1787826268.045825:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:3.0:1787826268.045827:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:3.0:1787826268.045829:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:3.0:1787826268.045831:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:3.0:1787826268.045833:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:3.0:1787826268.045835:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:3.0:1787826268.045837:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:3.0:1787826268.045839:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:3.0:1787826268.045841:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:3.0:1787826268.045843:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:3.0:1787826268.045845:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:3.0:1787826268.045846:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:3.0:1787826268.045848:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:3.0:1787826268.045849:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:3.0:1787826268.045851:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:3.0:1787826268.045853:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:3.0:1787826268.045854:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:3.0:1787826268.045856:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:3.0:1787826268.045858:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:3.0:1787826268.045860:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:3.0:1787826268.045862:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:3.0:1787826268.045864:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:3.0:1787826268.045867:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:3.0:1787826268.045869:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:3.0:1787826268.045871:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:3.0:1787826268.045872:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:3.0:1787826268.045874:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:3.0:1787826268.045876:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:3.0:1787826268.045878:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:3.0:1787826268.045880:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:3.0:1787826268.045881:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:3.0:1787826268.045883:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:3.0:1787826268.045886:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:3.0:1787826268.045888:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:3.0:1787826268.045890:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:3.0:1787826268.045892:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:3.0:1787826268.045894:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:3.0:1787826268.045896:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:3.0:1787826268.045898:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:3.0:1787826268.045900:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:3.0:1787826268.045902:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:3.0:1787826268.045904:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:3.0:1787826268.045906:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:3.0:1787826268.045908:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:3.0:1787826268.045910:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:3.0:1787826268.045911:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:3.0:1787826268.045913:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:3.0:1787826268.045915:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:3.0:1787826268.045916:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:3.0:1787826268.045918:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:3.0:1787826268.045920:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:3.0:1787826268.045922:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:3.0:1787826268.045924:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:3.0:1787826268.045926:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:3.0:1787826268.045928:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:3.0:1787826268.045930:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:3.0:1787826268.045932:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:3.0:1787826268.045933:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:3.0:1787826268.045935:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:3.0:1787826268.045937:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:3.0:1787826268.045939:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:3.0:1787826268.045941:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:3.0:1787826268.045943:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:3.0:1787826268.045945:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:3.0:1787826268.045947:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:3.0:1787826268.045950:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:3.0:1787826268.045952:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:3.0:1787826268.045955:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:3.0:1787826268.045957:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:3.0:1787826268.045959:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:3.0:1787826268.045961:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:3.0:1787826268.045963:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:3.0:1787826268.045965:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:3.0:1787826268.045966:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:3.0:1787826268.045968:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:3.0:1787826268.045970:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:3.0:1787826268.045972:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:3.0:1787826268.045974:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:3.0:1787826268.045976:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:3.0:1787826268.045978:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:3.0:1787826268.045980:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:3.0:1787826268.045982:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:3.0:1787826268.045984:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:3.0:1787826268.045986:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:3.0:1787826268.045989:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:3.0:1787826268.045991:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:3.0:1787826268.045993:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:3.0:1787826268.045996:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:3.0:1787826268.045998:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:3.0:1787826268.046000:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:3.0:1787826268.046002:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:3.0:1787826268.046004:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:3.0:1787826268.046006:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:3.0:1787826268.046008:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:3.0:1787826268.046009:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:3.0:1787826268.046012:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:3.0:1787826268.046014:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:3.0:1787826268.046016:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:3.0:1787826268.046019:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:3.0:1787826268.046021:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:3.0:1787826268.046024:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:3.0:1787826268.046026:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:3.0:1787826268.046028:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:3.0:1787826268.046030:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:3.0:1787826268.046032:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:3.0:1787826268.046034:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:3.0:1787826268.046036:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:3.0:1787826268.046038:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:3.0:1787826268.046040:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:3.0:1787826268.046042:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:3.0:1787826268.046044:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:3.0:1787826268.046058:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:3.0:1787826268.046060:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:3.0:1787826268.046062:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:3.0:1787826268.046064:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:3.0:1787826268.046066:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:3.0:1787826268.046069:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:3.0:1787826268.046071:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:3.0:1787826268.046073:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:3.0:1787826268.046075:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:3.0:1787826268.046077:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:3.0:1787826268.046079:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:3.0:1787826268.046081:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:3.0:1787826268.046083:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:3.0:1787826268.046085:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:3.0:1787826268.046087:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:3.0:1787826268.046089:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:3.0:1787826268.046090:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:3.0:1787826268.046093:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:3.0:1787826268.046095:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:3.0:1787826268.046107:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:3.0:1787826268.046109:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:3.0:1787826268.046111:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:3.0:1787826268.046113:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:3.0:1787826268.046115:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:3.0:1787826268.046117:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:3.0:1787826268.046119:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:3.0:1787826268.046121:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:3.0:1787826268.046123:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:3.0:1787826268.046125:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:3.0:1787826268.046127:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:3.0:1787826268.046129:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:3.0:1787826268.046131:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:3.0:1787826268.046134:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:3.0:1787826268.046136:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:3.0:1787826268.046138:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:3.0:1787826268.046140:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:3.0:1787826268.046142:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:3.0:1787826268.046144:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:3.0:1787826268.046146:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:3.0:1787826268.046148:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:3.0:1787826268.046150:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:3.0:1787826268.046152:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:3.0:1787826268.046154:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:3.0:1787826268.046156:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:3.0:1787826268.046158:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:3.0:1787826268.046160:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:3.0:1787826268.046162:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:3.0:1787826268.046164:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:3.0:1787826268.046166:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:3.0:1787826268.046169:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:3.0:1787826268.046171:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:3.0:1787826268.046173:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:3.0:1787826268.046175:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:3.0:1787826268.046177:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:3.0:1787826268.046179:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:3.0:1787826268.046181:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:3.0:1787826268.046183:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:3.0:1787826268.046185:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:3.0:1787826268.046187:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:3.0:1787826268.046189:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:3.0:1787826268.046191:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:3.0:1787826268.046193:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:3.0:1787826268.046195:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:3.0:1787826268.046197:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:3.0:1787826268.046199:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:3.0:1787826268.046202:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:3.0:1787826268.046204:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:3.0:1787826268.046206:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:3.0:1787826268.046208:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:3.0:1787826268.046211:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:3.0:1787826268.046212:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:3.0:1787826268.046215:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:3.0:1787826268.046217:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:3.0:1787826268.046219:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:3.0:1787826268.046221:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:3.0:1787826268.046223:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:3.0:1787826268.046225:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:3.0:1787826268.046227:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:3.0:1787826268.046229:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:3.0:1787826268.046230:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:3.0:1787826268.046232:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:3.0:1787826268.046234:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:3.0:1787826268.048239:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:3.0:1787826268.048364:0:2351:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907da56f8b40 00000100:00100000:3.0:1787826268.048412:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011414144:192.168.203.104@tcp:4:lfs.0 00000080:00200000:0.0:1787826268.052418:0:66551:0:(file.c:2207:ll_file_io_generic()) Range unlock [0, 511] 00000080:00200000:0.0:1787826268.052449:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1787826268.052456:0:66551:0:(file.c:2240:ll_file_io_generic()) f204b.sanity-flr: 2 io complete with rc: 0, result: 2097152, restart: 0 00000080:00200000:1.0:1787826268.052883:0:66551:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x200:0x0] updating i_size 2097152 i_blocks 4096 00000020:00200000:1.0:1787826268.052891:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826268.052910:0:66551:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 2097152 00000080:00200080:1.0:1787826268.052937:0:66551:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248) cmd=400866b1 arg=0 00000080:00200080:1.0:1787826268.052949:0:66551:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248) cmd=400866b1 arg=1 00000080:00200008:1.0:1787826268.052957:0:66551:0:(file.c:6313:ll_getattr_dentry()) START file f204b.sanity-flr:[0x200000404:0x200:0x0](ffff907d93313248), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826268.052967:0:66551:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248),name=f204b.sanity-flr 00000002:00010000:1.0:1787826268.052995:0:66551:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x200:0x0]) in obj [0x200000404:0x200:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826268.053021:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826268.053038:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1787826268.054230:0:66551:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x200:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826268.054244:0:66551:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) 00000080:00010000:1.0:1787826268.054249:0:66551:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) for lock 0xe5f736940357b7d9 00000080:00010000:1.0:1787826268.054261:0:66551:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b7d9 from it ffff9dd141cdbcd8 00010000:00010000:1.0:1787826268.054266:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826268.054278:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1787826268.054393:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:1.0:1787826268.054409:0:66551:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826268.054413:0:66551:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826268.054418:0:66551:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x200:0x0] 00000008:00010000:1.0:1787826268.054447:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826268.054451:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88fe80f0, osc lock ffff907d962f4ea0, flags 0x400000400001002 00000008:00010000:1.0:1787826268.054459:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826268.054460:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88fe8130, osc lock ffff907d962f4270, flags 0x400000400001002 00010000:00010000:1.0:1787826268.054484:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826268.054498:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00200000:1.0:1787826268.054523:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826268.054528:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826268.054537:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826268.054561:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826268.054572:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 68157439) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00200000:1.0:1787826268.054580:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826268.054583:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826268.054591:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000020:00200000:1.0:1787826268.054600:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826268.054627:0:66551:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x200:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:1.0:1787826268.054633:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_release():245 00000020:00010000:1.0:1787826268.054637:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826268.054641:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88fe80f0 (ffff907da5608b48/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826268.054645:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88fe8130 (ffff907da5608878/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826268.054648:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826268.054652:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88fe80f0 (ffff907d884a7f38/1) at cl_lock_fini():72 00000020:00010000:1.0:1787826268.054656:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88fe8130 (ffff907d884a7f38/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826268.054669:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826268.054694:0:66551:0:(file.c:5224:ll_file_seek()) START file f204b.sanity-flr:[0x200000404:0x200:0x0], offset: 2097152, type: SEEK_DATA 00000080:00200000:1.0:1787826268.054702:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:1.0:1787826268.054710:0:66551:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826268.054713:0:66551:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826268.054718:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:1.0:1787826268.054724:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:1.0:1787826268.054741:0:66551:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x200:0x0] updating i_size 2097152 i_blocks 4096 00000080:00200000:1.0:1787826268.054752:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826268.054758:0:66551:0:(file.c:1848:ll_io_set_mirror()) f204b.sanity-flr: desiginated mirror: 1 00000080:00200000:1.0:1787826268.054760:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826268.054765:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:1.0:1787826268.054768:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:1.0:1787826268.054772:0:66551:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [512, 18446744073709551615] 00000008:00010000:1.0:1787826268.054779:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:1.0:1787826268.054782:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c32fb0, osc lock ffff907d962f4270, flags 0x0 00010000:00010000:1.0:1787826268.054791:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826268.054801:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (2097152 68157439) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826268.054810:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826268.054817:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826268.054824:0:66551:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1787826268.054833:0:66551:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x200:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:1.0:1787826268.054839:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_release():245 00000020:00010000:1.0:1787826268.054842:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826268.054844:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c32fb0 (ffff907da5608878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1787826268.054848:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826268.054857:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000020:00010000:1.0:1787826268.054866:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826268.054869:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c32fb0 (ffff907d884a7f38/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826268.054876:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826268.054880:0:66551:0:(file.c:5262:ll_file_seek()) COMPLETED file f204b.sanity-flr:[0x200000404:0x200:0x0], offset: 2097152, type: SEEK_DATA, rc = -6 00000080:00200008:1.0:1787826268.054887:0:66551:0:(file.c:5224:ll_file_seek()) START file f204b.sanity-flr:[0x200000404:0x200:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1787826268.054891:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:1.0:1787826268.054896:0:66551:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826268.054898:0:66551:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826268.054900:0:66551:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x200:0x0] 00000008:00010000:1.0:1787826268.054906:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826268.054908:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88fe80f0, osc lock ffff907d962f4270, flags 0x400000400001002 00000008:00010000:1.0:1787826268.054912:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826268.054913:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88fe8130, osc lock ffff907d962f4ea0, flags 0x400000400001002 00010000:00010000:1.0:1787826268.054921:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826268.054931:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000020:00200000:1.0:1787826268.054946:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826268.054948:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826268.054956:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1787826268.054969:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1787826268.054977:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 68157439) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000020:00200000:1.0:1787826268.054985:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826268.054988:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1787826268.054996:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00000020:00200000:1.0:1787826268.055004:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826268.055009:0:66551:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x200:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:1.0:1787826268.055012:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_release():245 00000020:00010000:1.0:1787826268.055014:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826268.055017:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88fe80f0 (ffff907da5608878/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826268.055021:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88fe8130 (ffff907da5608b48/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826268.055023:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826268.055026:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88fe80f0 (ffff907d884a7f38/1) at cl_lock_fini():72 00000020:00010000:1.0:1787826268.055029:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88fe8130 (ffff907d884a7f38/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826268.055035:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826268.055041:0:66551:0:(file.c:5262:ll_file_seek()) COMPLETED file f204b.sanity-flr:[0x200000404:0x200:0x0], offset: 0, type: (null), rc = 2097152 00000080:00200080:1.0:1787826268.055557:0:66551:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248) cmd=400866b1 arg=0 00000080:00200080:1.0:1787826268.075228:0:66551:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248) cmd=400866b1 arg=2 00000080:00200008:1.0:1787826268.075257:0:66551:0:(file.c:6313:ll_getattr_dentry()) START file f204b.sanity-flr:[0x200000404:0x200:0x0](ffff907d93313248), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826268.075269:0:66551:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248),name=f204b.sanity-flr 00000002:00010000:1.0:1787826268.076042:0:66551:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x200:0x0]) in obj [0x200000404:0x200:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826268.076082:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826268.076101:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1787826268.076110:0:66551:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x200:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826268.076121:0:66551:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) 00000080:00010000:1.0:1787826268.076125:0:66551:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) for lock 0xe5f736940357b7d9 00000080:00010000:1.0:1787826268.076135:0:66551:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b7d9 from it ffff9dd141cdbcd8 00010000:00010000:1.0:1787826268.076140:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826268.076150:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1787826268.076195:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:1.0:1787826268.076213:0:66551:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826268.076217:0:66551:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826268.076222:0:66551:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x200:0x0] 00000008:00010000:1.0:1787826268.076251:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826268.076256:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88fe80f0, osc lock ffff907d962f4ea0, flags 0x400000400001002 00000008:00010000:1.0:1787826268.076265:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826268.076267:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88fe8130, osc lock ffff907d962f4270, flags 0x400000400001002 00010000:00010000:1.0:1787826268.076289:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1787826268.077178:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000020:00200000:1.0:1787826268.077198:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826268.077203:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1787826268.077212:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1787826268.077242:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1787826268.077252:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (1048576 68157439) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00000020:00200000:1.0:1787826268.077266:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826268.077270:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1787826268.077278:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00000020:00200000:1.0:1787826268.077287:0:66551:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826268.077367:0:66551:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x200:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:1.0:1787826268.077374:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_release():245 00000020:00010000:1.0:1787826268.077378:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826268.077381:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88fe80f0 (ffff907da5608b48/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826268.077386:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88fe8130 (ffff907da5608878/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826268.077390:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826268.077393:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88fe80f0 (ffff907d884a7f38/1) at cl_lock_fini():72 00000020:00010000:1.0:1787826268.077401:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88fe8130 (ffff907d884a7f38/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826268.077417:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826268.077475:0:66551:0:(llite_lib.c:2427:ll_setattr_raw()) START file f204b.sanity-flr:[0x200000404:0x200:0x0](ffff907d93313248) current size 2097152, valid attrs 2068, mode 0, uid -1976721614, gid -1, new size 2097152, atime 1787826268.000000000, mtime 1787826268.000000000, ctime 1787826268.000000000 00010000:00010000:1.0:1787826268.077522:0:66551:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826268.077545:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826268.077627:0:66551:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826268.077635:0:66551:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826268.077638:0:66551:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b7d9 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0fc19 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00000020:00100000:1.0:1787826268.077666:0:66551:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1787826268.077682:0:66551:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0000 x1874670011414400/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826268.077712:0:66551:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011414400:192.168.203.104@tcp:36:lfs.0 00000100:00100000:1.0:1787826268.077934:0:66551:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe80c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826268.097259:0:66551:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011414400:192.168.203.104@tcp:36:lfs.0 00000080:00200000:1.0:1787826268.097339:0:66551:0:(file.c:1848:ll_io_set_mirror()) f204b.sanity-flr: desiginated mirror: 2 00000080:00200000:1.0:1787826268.097348:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:1.0:1787826268.097378:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[3] flags 0x11 00020000:00200000:1.0:1787826268.097396:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[4] flags 0x11 00000080:00200000:1.0:1787826268.097407:0:66551:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1787826268.097425:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1787826268.097430:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88fe80f0, osc lock ffff907d962f4270, flags 0x0 00000008:00010000:1.0:1787826268.097435:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1787826268.097436:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88fe8130, osc lock ffff907d962f4ea0, flags 0x0 00010000:00010000:1.0:1787826268.097526:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357b7ee lrc: 3/0,1 mode: --/PW res: [0x280000400:0x309:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.097543:0:66551:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357b7ee lrc: 3/0,1 mode: --/PW res: [0x280000400:0x309:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.097562:0:66551:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357b7ee lrc: 3/0,1 mode: --/PW res: [0x280000400:0x309:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826268.097589:0:66551:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011414528:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826268.097736:0:66551:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe8c00 going to sleep for 11 seconds 00000100:00100000:1.0:1787826268.103341:0:66551:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011414528:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826268.103381:0:66551:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357b7ee lrc: 4/0,1 mode: --/PW res: [0x280000400:0x309:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc4a expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.103406:0:66551:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357b7ee lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x309:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fc4a expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.103420:0:66551:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357b7ee lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x309:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc4a expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.103439:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357b7ee lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x309:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fc4a expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826268.103455:0:66551:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357b7ee lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x309:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fc4a expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.103469:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357b7ee lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x309:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc4a expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.103484:0:66551:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357b7ee lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x309:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc4a expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.103602:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357b7f5 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30a:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.103662:0:66551:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357b7f5 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30a:0x0].0x0 rrc: 2 type: EXT [10485760->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.103674:0:66551:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357b7f5 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30a:0x0].0x0 rrc: 2 type: EXT [10485760->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826268.103701:0:66551:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011414656:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826268.103818:0:66551:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe8c00 going to sleep for 11 seconds 00000100:00100000:1.0:1787826268.107740:0:66551:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011414656:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826268.107781:0:66551:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357b7f5 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x30a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc51 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.107805:0:66551:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357b7f5 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fc51 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.107819:0:66551:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357b7f5 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc51 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.107844:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357b7f5 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x30a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fc51 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826268.107860:0:66551:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357b7f5 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x30a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fc51 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.108073:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357b7f5 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x30a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc51 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.108091:0:66551:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357b7f5 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc51 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826268.108656:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011414912:192.168.203.104@tcp:10:lfs.0 00000100:00100000:1.0:1787826268.108793:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1787826268.108800:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011414784:192.168.203.104@tcp:10:lfs.0 00000100:00100000:1.0:1787826268.114678:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011414784:192.168.203.104@tcp:10:lfs.0 00000100:00100000:1.0:1787826268.115541:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011414912:192.168.203.104@tcp:10:lfs.0 00000080:00200000:0.0:1787826268.115604:0:66551:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x200:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:0.0:1787826268.115691:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_release():245 00000020:00010000:0.0:1787826268.115696:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1787826268.115702:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88fe80f0 (ffff907da5608878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1787826268.115715:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357b7ee lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x309:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc4a expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826268.115731:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357b7ee lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x309:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc4a expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:0.0:1787826268.115751:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88fe8130 (ffff907da5608b48/1) at cl_lock_cancel():142 00010000:00010000:0.0:1787826268.115757:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357b7f5 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc0fc51 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826268.115773:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357b7f5 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x30a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc51 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:0.0:1787826268.115780:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1787826268.115783:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88fe80f0 (ffff907d884a7f38/1) at cl_lock_fini():72 00000020:00010000:0.0:1787826268.115790:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88fe8130 (ffff907d884a7f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1787826268.115812:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1787826268.115828:0:66551:0:(llite_lib.c:2686:ll_setattr_raw()) COMPLETED file f204b.sanity-flr:[0x200000404:0x200:0x0](ffff907d93313248) current size 2097152, valid attrs 10002168, mode 0, uid -1976721614, gid -1, new size 2097152, atime 1787826268.000000000, mtime 1787826268.000000000, ctime 1787826268.000000000, rc 0 00000080:00200080:0.0:1787826268.115854:0:66551:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248) cmd=400866b1 arg=0 00000080:00200008:0.0:1787826268.115965:0:66551:0:(llite_lib.c:2427:ll_setattr_raw()) START file f204b.sanity-flr:[0x200000404:0x200:0x0](ffff907d93313248) current size 2097152, valid attrs 101f0, mode 9dd1, uid -2113587520, gid -28547, new size 18446621467747627592, atime 1787826267.000000000, mtime 1787826267.000000000, ctime 1787826268.000000000 00000020:00100000:0.0:1787826268.116041:0:66551:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1787826268.116051:0:66551:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908461c0 x1874670011415168/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787826268.116081:0:66551:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011415168:192.168.203.104@tcp:36:lfs.0 00000100:00100000:0.0:1787826268.116236:0:66551:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fdd80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826268.118766:0:66551:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011415168:192.168.203.104@tcp:36:lfs.0 00000080:00200000:0.0:1787826268.118826:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:0.0:1787826268.118845:0:66551:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826268.118849:0:66551:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826268.118859:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826268.118887:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:0.0:1787826268.118898:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:0.0:1787826268.118903:0:66551:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:0.0:1787826268.118923:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1787826268.118929:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d998fddb0, osc lock ffff907d963684e0, flags 0x0 00000008:00010000:0.0:1787826268.118934:0:66551:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1787826268.118935:0:66551:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d998fddf0, osc lock ffff907d96368c30, flags 0x0 00010000:00010000:0.0:1787826268.118957:0:66551:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d91e6e400 type 11 mode 2 res 9663677440/804 (0 1048575) 00010000:00010000:0.0:1787826268.119016:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 3/0,1 mode: --/PW res: [0x240000400:0x324:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826268.119031:0:66551:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 3/0,1 mode: --/PW res: [0x240000400:0x324:0x0].0x0 rrc: 3 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826268.119043:0:66551:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 3/0,1 mode: --/PW res: [0x240000400:0x324:0x0].0x0 rrc: 3 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826268.119067:0:66551:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011415296:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826268.119456:0:66551:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd180 going to sleep for 11 seconds 00000100:00100000:2.0:1787826268.123438:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826268.123586:0:50445:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:3.0:1787826268.123722:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025486080 00000100:00100000:3.0:1787826268.123735:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025486080 00000100:00100000:2.0:1787826268.123996:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7136 00000100:00100000:2.0:1787826268.124011:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d88635c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:6934:x1874670025486080:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826268.124035:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:2.0:1787826268.124148:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826268.124156:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d88635c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:6934:x1874670025486080:12345-192.168.203.104@tcp:104: Request processed in 150us (734us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826268.124165:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7136 00010000:00010000:1.0:1787826268.124250:0:40346:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826268.124260:0:40346:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1787826268.124268:0:40346:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883b400 already unused, calling callback (ffffffffc118e410) 00010000:00010000:1.0:1787826268.124275:0:40346:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x324:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800428400000000 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00000080:00200000:1.0:1787826268.124340:0:40346:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1787826268.124357:0:40346:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1787826268.124418:0:40346:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 2/0,0 mode: --/PR res: [0x240000400:0x324:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1787826268.124426:0:40346:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826268.124463:0:40346:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x324:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1787826268.124472:0:40346:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000006e4b3f7c/0xe5f736940357b7e0 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x324:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0fc20 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1787826268.124490:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826268.124492:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826268.124494:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826268.124515:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1787826268.124533:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011415424:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1787826268.134956:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826268.135116:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025486208 00000100:00100000:3.0:1787826268.135128:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025486208 00000100:00100000:3.0:1787826268.135192:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7137 00000100:00100000:3.0:1787826268.135204:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91148000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:28562:x1874670025486208:12345-192.168.203.104@tcp:105: 00010000:00010000:3.0:1787826268.135225:0:40339:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 4/0,1 mode: --/PW res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc5f expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.135242:0:40339:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 4/0,1 mode: --/PW res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc5f expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.135253:0:40339:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 4/0,1 mode: --/PW res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fc5f expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.135267:0:40339:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10080000000000 nid: local remote: 0x7918213f4fc0fc5f expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.135279:0:40339:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10080000000000 nid: local remote: 0x7918213f4fc0fc5f expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.135291:0:40339:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x80000000000 nid: local remote: 0x7918213f4fc0fc5f expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.135325:0:40339:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff907d8883ba00) 00000100:00100000:3.0:1787826268.135400:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91148000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:28562:x1874670025486208:12345-192.168.203.104@tcp:105: Request processed in 198us (458us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826268.135408:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7137 00000100:00100000:0.0:1787826268.136827:0:66551:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011415296:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826268.136859:0:66551:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0fc5f expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826268.136884:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc0fc5f expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826268.136899:0:66551:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 1048576, setting rss=1048576; kms=1048576, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10020000020000 nid: local remote: 0x7918213f4fc0fc5f expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826268.136920:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc0fc5f expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826268.136929:0:66551:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc0fc5f expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826268.136968:0:66551:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d91e6e400 type 11 mode 2 res 9663677440/805 (1048576 68157439) 00010000:00010000:0.0:1787826268.137017:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826268.137026:0:66551:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [1048576->68157439] (req 1048576->68157439) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826268.137037:0:66551:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [1048576->68157439] (req 1048576->68157439) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826268.137061:0:66551:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011415552:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826268.138919:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011415424:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826268.139163:0:66551:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd180 going to sleep for 11 seconds 00000100:00100000:3.0:1787826268.141595:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826268.142254:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025486336 00000100:00100000:3.0:1787826268.142267:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025486336 00000100:00100000:3.0:1787826268.142344:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7138 00000100:00100000:3.0:1787826268.142355:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83525680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:35506:x1874670025486336:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826268.142379:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00010000:00010000:3.0:1787826268.142670:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826268.142678:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83525680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:35506:x1874670025486336:12345-192.168.203.104@tcp:104: Request processed in 324us (1099us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826268.142686:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7138 00000100:00100000:1.0:1787826268.144723:0:66551:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011415552:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826268.144765:0:66551:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [1048576->68157439] (req 1048576->68157439) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.144794:0:66551:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [1048576->68157439] (req 1048576->68157439) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.144806:0:66551:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [1048576->68157439] (req 1048576->68157439) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826268.144884:0:60482:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826268.144894:0:60482:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00010000:00010000:0.0:1787826268.144905:0:60482:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883b200 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826268.144912:0:60482:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800428400000000 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00000080:00200000:0.0:1787826268.144938:0:60482:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1787826268.144956:0:60482:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1787826268.145017:0:60482:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 2/0,0 mode: --/PR res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00010000:00010000:0.0:1787826268.145026:0:60482:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:1.0:1787826268.145114:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011415680:192.168.203.104@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1787826268.145781:0:60482:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00010000:00010000:0.0:1787826268.145794:0:60482:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b3f0f3c6/0xe5f736940357b7e7 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc0fc27 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00010000:00010000:0.0:1787826268.145815:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826268.145817:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826268.145819:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826268.150279:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826268.150410:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025486464 00000100:00100000:2.0:1787826268.150421:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025486464 00000100:00100000:2.0:1787826268.150486:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7139 00000100:00100000:2.0:1787826268.150497:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83526580 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:5933:x1874670025486464:12345-192.168.203.104@tcp:105: 00010000:00010000:2.0:1787826268.150526:0:40338:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 5/0,1 mode: --/PW res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [1048576->68157439] (req 1048576->68157439) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826268.150543:0:40338:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 5/0,1 mode: --/PW res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [1048576->68157439] (req 1048576->68157439) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826268.150554:0:40338:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 5/0,1 mode: --/PW res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826268.150567:0:40338:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x10080000020000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826268.150579:0:40338:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x10080000020000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826268.150592:0:40338:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x80000020000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826268.150653:0:40338:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff907d8883a200) 00000100:00100000:2.0:1787826268.150721:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83526580 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:5933:x1874670025486464:12345-192.168.203.104@tcp:105: Request processed in 227us (457us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826268.150729:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7139 00010000:00010000:0.0:1787826268.153022:0:66551:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826268.153050:0:66551:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826268.153073:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826268.153102:0:66551:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x10020000020000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826268.153118:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826268.153126:0:66551:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826268.153692:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1787826268.153715:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011415808:192.168.203.104@tcp:2:lfs.0 00000100:00100000:0.0:1787826268.154548:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1787826268.154565:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011415936:192.168.203.104@tcp:2:lfs.0 00000100:00100000:1.0:1787826268.164678:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011415808:192.168.203.104@tcp:2:lfs.0 00000100:00100000:1.0:1787826268.164701:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011415680:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826268.164889:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011415936:192.168.203.104@tcp:2:lfs.0 00000020:00010000:2.0:1787826268.164949:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_release():245 00000020:00010000:2.0:1787826268.164955:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826268.164962:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d998fddb0 (ffff907da5608b48/1) at cl_lock_cancel():142 00010000:00010000:2.0:1787826268.164976:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc0fc5f expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826268.164994:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0fc5f expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1787826268.165016:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d998fddf0 (ffff907da5608878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1787826268.165023:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826268.165030:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1787826268.165037:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826268.165040:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d998fddb0 (ffff907d884a7f38/1) at cl_lock_fini():72 00000020:00010000:2.0:1787826268.165046:0:66551:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d998fddf0 (ffff907d884a7f38/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826268.165088:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826268.165112:0:66551:0:(llite_lib.c:2686:ll_setattr_raw()) COMPLETED file f204b.sanity-flr:[0x200000404:0x200:0x0](ffff907d93313248) current size 2097152, valid attrs 100101f0, mode 9dd1, uid -2113587520, gid -28547, new size 18446621467747627592, atime 1787826267.000000000, mtime 1787826267.000000000, ctime 1787826268.000000000, rc 0 00000080:00200080:2.0:1787826268.165142:0:66551:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248) cmd=c00c66f3 arg=8892a0 00010000:00010000:2.0:1787826268.165207:0:66551:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a200/0xe5f736940357b7cb lrc: 2/0,0 mode: PW/PW res: [0x200000404:0x200:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x81008000000000 nid: local remote: 0x7918213f4fc0fbe8 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826268.165228:0:66551:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639a200/0xe5f736940357b7cb lrc: 0/0,0 mode: --/PW res: [0x200000404:0x200:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x85809400000000 nid: local remote: 0x7918213f4fc0fbe8 expref: -99 pid: 66551 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826268.165254:0:66551:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d9114ad00 x1874670011411968/t8589942965(8589942965) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826285 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1787826268.165267:0:66551:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1787826268.165273:0:66551:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114a940 x1874670011416192/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 504/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1787826268.165312:0:66551:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011416192:192.168.203.104@tcp:35:lfs.0 00000100:00100000:2.0:1787826268.165411:0:66551:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bbd80 going to sleep for 11 seconds 00000100:00100000:3.0:1787826268.172927:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826268.173083:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025486592 00000100:00100000:2.0:1787826268.173096:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025486592 00000100:00100000:2.0:1787826268.173169:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7140 00000100:00100000:2.0:1787826268.173180:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83526940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:35517:x1874670025486592:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826268.173205:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826268.173316:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826268.173323:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83526940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:35517:x1874670025486592:12345-192.168.203.104@tcp:104: Request processed in 146us (412us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826268.173335:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7140 00010000:00010000:1.0:1787826268.173469:0:60835:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826268.173479:0:60835:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826268.173488:0:60835:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a000 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826268.173493:0:60835:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826268.173502:0:60835:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826268.173520:0:60835:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 3/0,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1787826268.173552:0:60835:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x200:0x0]: losing layout lock 00010000:00010000:1.0:1787826268.173604:0:60835:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 2/0,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826268.173662:0:60835:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826268.173685:0:60835:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 1/0,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826268.173693:0:60835:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b7d2 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0fc0b expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826268.173713:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826268.173714:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826268.173716:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826268.173725:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1787826268.173743:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011416320:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826268.178708:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011416320:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1787826268.190205:0:66551:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589942970 gen 1 00000100:00100000:2.0:1787826268.190258:0:66551:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011416192:192.168.203.104@tcp:35:lfs.0 00000002:00100000:2.0:1787826268.190303:0:66551:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d9114ad00 x1874670011411968/t8589942965(8589942965) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1787826268.190352:0:66551:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 31 res 8589935620/512 (0 0) 00000080:00010000:2.0:1787826268.190368:0:66551:0:(file.c:7076:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000404:0x200:0x0](ffff907d93313248) 00000002:00010000:2.0:1787826268.190385:0:66551:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x200:0x0]) in obj [0x200000404:0x200:0x0], intent: layout flags 0 00010000:00010000:2.0:1787826268.190451:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 3/1,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66551 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826268.190461:0:66551:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 3/1,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66551 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826268.190472:0:66551:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 3/1,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66551 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826268.190502:0:66551:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011416448:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826268.190638:0:66551:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bbd80 going to sleep for 13 seconds 00000100:00100000:2.0:1787826268.198274:0:66551:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011416448:192.168.203.104@tcp:101:lfs.0 00010000:00010000:2.0:1787826268.198327:0:66551:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fc74 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826268.198344:0:66551:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc74 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826268.198366:0:66551:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x400 disposition=0x1, status=0 req@ffff907d9114a940 x1874670011416448/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/896 e 0 to 0 dl 1787826286 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1787826268.198378:0:66551:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc74 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1787826268.198403:0:66551:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc74 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826268.198418:0:66551:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/512 (0 0) 00000080:00200000:2.0:1787826268.198425:0:66551:0:(namei.c:952:ll_intent_lock()) intent lock 1024 on i1 [0x200000404:0x200:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1787826268.198446:0:66551:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) for lock 0xe5f736940357b80a 00000080:00010000:2.0:1787826268.198456:0:66551:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b80a from it ffff9dd141cdbbc8 00010000:00010000:2.0:1787826268.198460:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fc74 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826268.198469:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc74 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826268.198498:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc74 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826268.198520:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc74 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1787826268.198529:0:66551:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x200:0x0](000000001c1f1433) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc74 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826268.198587:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fc74 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826268.198594:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc74 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1787826268.198648:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1787826268.198661:0:66551:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826268.198665:0:66551:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1787826268.198672:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1787826268.198684:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1787826268.198690:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:2.0:1787826268.198718:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1787826268.198730:0:66551:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc74 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826268.198739:0:66551:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc74 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1787826268.198746:0:66551:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x200:0x0](000000001c1f1433) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fc74 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1787826268.198765:0:66551:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1787826268.198770:0:66551:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826268.198772:0:66551:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1787826268.198774:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1787826268.198777:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1787826268.198782:0:66551:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:2.0:1787826268.198789:0:66551:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:2.0:1787826268.198815:0:66551:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 1048576 blocks 2048 [cma]time: 1787826267 1787826267 0 by oinfo size 1048576 blocks 2048 [cma]time 1787826268 1787826267 1787826267 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x324:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0fc5f expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:2.0:1787826268.198846:0:66551:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 2097152 blocks 2048 [cma]time: 1787826267 1787826267 0 by oinfo size 2097152 blocks 2048 [cma]time 1787826268 1787826267 1787826267 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x325:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:2.0:1787826268.198863:0:66551:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 2097152 blocks 4096 [cma]time: 1787826267 1787826267 0 by oinfo size 2097152 blocks 4096 [cma]time 1787826268 1787826268 0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357b7ee lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x309:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc4a expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:2.0:1787826268.198879:0:66551:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 0 blocks 0 [cma]time 1787826268 1787826268 0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357b7f5 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x30a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc0fc51 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826268.198941:0:66551:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x200:0x0]: layout version change: 4294967294 -> 8 00010000:00010000:2.0:1787826268.198946:0:66551:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fc74 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826268.198954:0:66551:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fc74 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:2.0:1787826268.198992:0:66551:0:(file.c:479:ll_file_release()) START file f204b.sanity-flr:[0x200000404:0x200:0x0](ffff907d93313248), flags 154002 00000002:00100000:2.0:1787826268.199040:0:66551:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d835261c0 x1874670011411584/t8589942964(8589942964) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826285 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1787826268.199057:0:66551:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1787826268.199062:0:66551:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114a940 x1874670011416704/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1787826268.199087:0:66551:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011416704:192.168.203.104@tcp:35:lfs.0 00000100:00100000:2.0:1787826268.199188:0:66551:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bbd80 going to sleep for 11 seconds 00000100:00100000:2.0:1787826268.204077:0:66551:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66551:1874670011416704:192.168.203.104@tcp:35:lfs.0 00000002:00100000:2.0:1787826268.204123:0:66551:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d835261c0 x1874670011411584/t8589942964(8589942964) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826285 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826268.228895:0:66553:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826268.228934:0:66553:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826268.228958:0:66553:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:3.0:1787826268.228979:0:66553:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:3.0:1787826268.228988:0:66553:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826268.228997:0:66553:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826268.229001:0:66553:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:3.0:1787826268.229010:0:66553:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141853bf8 00010000:00010000:3.0:1787826268.229014:0:66553:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:3.0:1787826268.229024:0:66553:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00000080:00200000:3.0:1787826268.229042:0:66553:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826268.229051:0:66553:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=27 00000080:00200000:3.0:1787826268.229057:0:66553:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:3.0:1787826268.229070:0:66553:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), file ffff907d85152480, open_flags 18800, mode 0 opened 0 00000080:00200000:3.0:1787826268.229078:0:66553:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=open 00000002:00010000:3.0:1787826268.229095:0:66553:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1ff:0x0], intent: open flags 0304001 00010000:00010000:3.0:1787826268.229170:0:66553:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eea00/0xe5f736940357b811 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 66553 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.229181:0:66553:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eea00/0xe5f736940357b811 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66553 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826268.229195:0:66553:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826268.229211:0:66553:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d835270c0 x1874670011416960/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826268.229221:0:66553:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eea00/0xe5f736940357b811 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66553 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826268.229249:0:66553:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66553:1874670011416960:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826268.229734:0:66553:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7d180 going to sleep for 13 seconds 00000100:00100000:1.0:1787826268.237080:0:66553:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66553:1874670011416960:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826268.237115:0:66553:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eea00/0xe5f736940357b811 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66553 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.237132:0:66553:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eea00/0xe5f736940357b811 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66553 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.237140:0:66553:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eea00/0xe5f736940357b811 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66553 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.237149:0:66553:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eea00/0xe5f736940357b811 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66553 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.237163:0:66553:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eea00/0xe5f736940357b811 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66553 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.237171:0:66553:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907da59eea00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826268.237177:0:66553:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eea00/0xe5f736940357b811 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66553 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.237184:0:66553:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eea00/0xe5f736940357b811 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66553 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.237192:0:66553:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eea00/0xe5f736940357b811 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 66553 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.237201:0:66553:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826268.237210:0:66553:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eea00/0xe5f736940357b811 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66553 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.237218:0:66553:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eea00/0xe5f736940357b811 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66553 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826268.237244:0:66553:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d835270c0 x1874670011416960/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826286 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826268.237260:0:66553:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1ff:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826268.238736:0:66553:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826268.238771:0:66553:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826268.238795:0:66553:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00010000:00010000:1.0:1787826268.238825:0:66553:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00000080:00200000:1.0:1787826268.238833:0:66553:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826268.238841:0:66553:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826268.238845:0:66553:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:1.0:1787826268.238854:0:66553:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141853bf8 00010000:00010000:1.0:1787826268.238858:0:66553:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00010000:00010000:1.0:1787826268.238867:0:66553:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000080:00200000:1.0:1787826268.238885:0:66553:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826268.238895:0:66553:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=267 00000080:00200000:1.0:1787826268.238904:0:66553:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 44 00000080:00200008:1.0:1787826268.238915:0:66553:0:(file.c:1090:ll_file_open()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00000080:00200080:1.0:1787826268.238965:0:66553:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8) cmd=c0086915 arg=8893d0 00000100:00100000:1.0:1787826268.239049:0:66553:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66553:1874670011417088:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826268.239179:0:66553:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe8c00 going to sleep for 13 seconds 00000100:00100000:2.0:1787826268.243374:0:66553:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66553:1874670011417088:192.168.203.104@tcp:34:lfs.0 00000080:00200008:2.0:1787826268.244279:0:66553:0:(file.c:479:ll_file_release()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00010000:00010000:2.0:1787826268.244324:0:66553:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fb01 expref: -99 pid: 66528 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200008:0.0:1787826268.272916:0:66555:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1787826268.272930:0:66555:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826268.272957:0:66555:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826268.272982:0:66555:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:0.0:1787826268.273000:0:66555:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000080:00200000:0.0:1787826268.273009:0:66555:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826268.273019:0:66555:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826268.273023:0:66555:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826268.273033:0:66555:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141c2fcb8 00010000:00010000:0.0:1787826268.273037:0:66555:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:0.0:1787826268.273053:0:66555:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00000080:00200000:0.0:1787826268.273192:0:66555:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826268.273201:0:66555:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826268.273209:0:66555:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00010000:00010000:0.0:1787826268.273219:0:66555:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00000080:00200000:0.0:1787826268.273225:0:66555:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826268.273229:0:66555:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826268.273233:0:66555:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826268.273237:0:66555:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141c2fbb0 00010000:00010000:0.0:1787826268.273240:0:66555:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00010000:00010000:0.0:1787826268.273247:0:66555:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00000080:00200000:0.0:1787826268.273257:0:66555:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:0.0:1787826268.273266:0:66555:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:0.0:1787826268.273286:0:66555:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffe1c5358bc 00000080:00200008:0.0:1787826268.273303:0:66555:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:2.0:1787826268.280350:0:66559:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826268.280390:0:66559:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826268.280410:0:66559:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:2.0:1787826268.280427:0:66559:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00000080:00200000:2.0:1787826268.280435:0:66559:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826268.280444:0:66559:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826268.280447:0:66559:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:2.0:1787826268.280455:0:66559:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141d3bbf8 00010000:00010000:2.0:1787826268.280460:0:66559:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:2.0:1787826268.280470:0:66559:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00000080:00200000:2.0:1787826268.280485:0:66559:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826268.280493:0:66559:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=27 00000080:00200000:2.0:1787826268.280499:0:66559:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:2.0:1787826268.280520:0:66559:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), file ffff907d934f2c00, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1787826268.280527:0:66559:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=open 00000002:00010000:2.0:1787826268.280540:0:66559:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1ff:0x0], intent: open flags 0304001 00010000:00010000:2.0:1787826268.280597:0:66559:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812ab000/0xe5f736940357b818 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 66559 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826268.280630:0:66559:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812ab000/0xe5f736940357b818 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66559 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826268.280642:0:66559:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826268.280655:0:66559:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114a940 x1874670011417344/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826268.280665:0:66559:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812ab000/0xe5f736940357b818 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66559 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826268.280687:0:66559:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66559:1874670011417344:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826268.280900:0:66559:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bbd80 going to sleep for 13 seconds 00000100:00100000:3.0:1787826268.288452:0:66559:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66559:1874670011417344:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826268.288484:0:66559:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812ab000/0xe5f736940357b818 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66559 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.288501:0:66559:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812ab000/0xe5f736940357b818 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66559 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.288519:0:66559:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812ab000/0xe5f736940357b818 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66559 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.288527:0:66559:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812ab000/0xe5f736940357b818 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66559 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.288537:0:66559:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812ab000/0xe5f736940357b818 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66559 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.288543:0:66559:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d812ab000 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826268.288549:0:66559:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812ab000/0xe5f736940357b818 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66559 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.288556:0:66559:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812ab000/0xe5f736940357b818 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66559 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.288563:0:66559:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812ab000/0xe5f736940357b818 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 66559 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.288571:0:66559:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826268.288580:0:66559:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812ab000/0xe5f736940357b818 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66559 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.288588:0:66559:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812ab000/0xe5f736940357b818 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66559 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826268.288757:0:66559:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d9114a940 x1874670011417344/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826286 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826268.288774:0:66559:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1ff:0x0] suppgids 0 -1: rc -20 00000080:00200000:3.0:1787826268.292879:0:66559:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826268.292919:0:66559:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826268.292944:0:66559:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00010000:00010000:3.0:1787826268.292964:0:66559:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00000080:00200000:3.0:1787826268.292973:0:66559:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826268.292981:0:66559:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826268.292985:0:66559:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:3.0:1787826268.292995:0:66559:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141d3bbf8 00010000:00010000:3.0:1787826268.292999:0:66559:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00010000:00010000:3.0:1787826268.293008:0:66559:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00000080:00200000:3.0:1787826268.293026:0:66559:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826268.293035:0:66559:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=267 00000080:00200000:3.0:1787826268.293043:0:66559:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 44 00000080:00200008:3.0:1787826268.293053:0:66559:0:(file.c:1090:ll_file_open()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00000080:00200080:3.0:1787826268.293097:0:66559:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8) cmd=c1406916 arg=980a20 00000100:00100000:3.0:1787826268.293186:0:66559:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66559:1874670011417472:192.168.203.104@tcp:34:lfs.0 00000100:00100000:3.0:1787826268.293327:0:66559:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7db40 going to sleep for 13 seconds 00000100:00100000:3.0:1787826268.297250:0:66559:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66559:1874670011417472:192.168.203.104@tcp:34:lfs.0 00000080:00200008:3.0:1787826268.299463:0:66559:0:(file.c:479:ll_file_release()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00010000:00010000:3.0:1787826268.299526:0:66559:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fb01 expref: -99 pid: 66528 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200008:1.0:1787826268.357910:0:66562:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1787826268.357921:0:66562:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826268.357943:0:66562:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826268.357961:0:66562:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00010000:00010000:1.0:1787826268.357978:0:66562:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00000080:00200000:1.0:1787826268.357992:0:66562:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826268.358001:0:66562:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826268.358004:0:66562:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:1.0:1787826268.358012:0:66562:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141d2bcb8 00010000:00010000:1.0:1787826268.358016:0:66562:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00010000:00010000:1.0:1787826268.358513:0:66562:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00000080:00200000:1.0:1787826268.358691:0:66562:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826268.358703:0:66562:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826268.358840:0:66562:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00010000:00010000:1.0:1787826268.358851:0:66562:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00000080:00200000:1.0:1787826268.358858:0:66562:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826268.358863:0:66562:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826268.358866:0:66562:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:1.0:1787826268.358872:0:66562:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141d2bbb0 00010000:00010000:1.0:1787826268.358874:0:66562:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00010000:00010000:1.0:1787826268.358881:0:66562:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00000080:00200000:1.0:1787826268.358891:0:66562:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:1.0:1787826268.358898:0:66562:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:1.0:1787826268.358917:0:66562:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffed449511c 00000080:00200008:1.0:1787826268.358929:0:66562:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:2.0:1787826268.364919:0:66566:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826268.364947:0:66566:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826268.364965:0:66566:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00010000:00010000:2.0:1787826268.364982:0:66566:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00000080:00200000:2.0:1787826268.364990:0:66566:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826268.364999:0:66566:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826268.365002:0:66566:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:2.0:1787826268.365010:0:66566:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141d9bbf8 00010000:00010000:2.0:1787826268.365015:0:66566:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00010000:00010000:2.0:1787826268.365023:0:66566:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00000080:00200000:2.0:1787826268.365037:0:66566:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826268.365044:0:66566:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=27 00000080:00200000:2.0:1787826268.365050:0:66566:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:2.0:1787826268.365063:0:66566:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), file ffff907da702db00, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1787826268.365070:0:66566:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=open 00000002:00010000:2.0:1787826268.365084:0:66566:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1ff:0x0], intent: open flags 0304001 00010000:00010000:2.0:1787826268.365151:0:66566:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa400/0xe5f736940357b81f lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 66566 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826268.365159:0:66566:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa400/0xe5f736940357b81f lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66566 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826268.365174:0:66566:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826268.365186:0:66566:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114ad00 x1874670011417728/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826268.365196:0:66566:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa400/0xe5f736940357b81f lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66566 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826268.365217:0:66566:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66566:1874670011417728:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826268.365556:0:66566:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb9c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826268.370476:0:66566:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66566:1874670011417728:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826268.370500:0:66566:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa400/0xe5f736940357b81f lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66566 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.370522:0:66566:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa400/0xe5f736940357b81f lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66566 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.370528:0:66566:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa400/0xe5f736940357b81f lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66566 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.370535:0:66566:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa400/0xe5f736940357b81f lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66566 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.370544:0:66566:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa400/0xe5f736940357b81f lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66566 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.370550:0:66566:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d812aa400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826268.370554:0:66566:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa400/0xe5f736940357b81f lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66566 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.370561:0:66566:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa400/0xe5f736940357b81f lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66566 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.370568:0:66566:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa400/0xe5f736940357b81f lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 66566 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.370575:0:66566:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826268.370583:0:66566:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa400/0xe5f736940357b81f lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66566 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.370590:0:66566:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa400/0xe5f736940357b81f lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66566 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826268.370832:0:66566:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d9114ad00 x1874670011417728/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826286 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826268.370847:0:66566:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1ff:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826268.371192:0:66566:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826268.371212:0:66566:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826268.371227:0:66566:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00010000:00010000:1.0:1787826268.371240:0:66566:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00000080:00200000:1.0:1787826268.371248:0:66566:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826268.371254:0:66566:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826268.371259:0:66566:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:1.0:1787826268.371265:0:66566:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141d9bbf8 00010000:00010000:1.0:1787826268.371269:0:66566:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00010000:00010000:1.0:1787826268.371279:0:66566:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00000080:00200000:1.0:1787826268.371301:0:66566:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826268.371309:0:66566:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=267 00000080:00200000:1.0:1787826268.371317:0:66566:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 44 00000080:00200008:1.0:1787826268.371326:0:66566:0:(file.c:1090:ll_file_open()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00000080:00200080:1.0:1787826268.371493:0:66566:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8) cmd=c1406916 arg=1b93a20 00000100:00100000:1.0:1787826268.371568:0:66566:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66566:1874670011417856:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826268.371692:0:66566:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe8c00 going to sleep for 13 seconds 00000100:00100000:1.0:1787826268.377016:0:66566:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66566:1874670011417856:192.168.203.104@tcp:34:lfs.0 00000080:00200008:2.0:1787826268.377823:0:66566:0:(file.c:479:ll_file_release()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00010000:00010000:2.0:1787826268.377852:0:66566:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fb01 expref: -99 pid: 66528 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200008:0.0:1787826268.432105:0:66569:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1787826268.432119:0:66569:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826268.432148:0:66569:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826268.432174:0:66569:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00010000:00010000:0.0:1787826268.432193:0:66569:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00000080:00200000:0.0:1787826268.432203:0:66569:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826268.432213:0:66569:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826268.432217:0:66569:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826268.432227:0:66569:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd14195bcb8 00010000:00010000:0.0:1787826268.432232:0:66569:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00010000:00010000:0.0:1787826268.432243:0:66569:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00000080:00200000:0.0:1787826268.432385:0:66569:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826268.432394:0:66569:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826268.432402:0:66569:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00010000:00010000:0.0:1787826268.432412:0:66569:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00000080:00200000:0.0:1787826268.432419:0:66569:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826268.432423:0:66569:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826268.432426:0:66569:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826268.432431:0:66569:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd14195bbb0 00010000:00010000:0.0:1787826268.432434:0:66569:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00010000:00010000:0.0:1787826268.432442:0:66569:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00000080:00200000:0.0:1787826268.432453:0:66569:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:0.0:1787826268.432462:0:66569:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:0.0:1787826268.432483:0:66569:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffe4839785c 00000080:00200008:0.0:1787826268.432498:0:66569:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:1.0:1787826268.440755:0:66573:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826268.440799:0:66573:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826268.440825:0:66573:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00010000:00010000:1.0:1787826268.440846:0:66573:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00000080:00200000:1.0:1787826268.440856:0:66573:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826268.440867:0:66573:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826268.440871:0:66573:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:1.0:1787826268.440881:0:66573:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141de3bf8 00010000:00010000:1.0:1787826268.440885:0:66573:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00010000:00010000:1.0:1787826268.440896:0:66573:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00000080:00200000:1.0:1787826268.440921:0:66573:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826268.440932:0:66573:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=27 00000080:00200000:1.0:1787826268.440939:0:66573:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:1.0:1787826268.440953:0:66573:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), file ffff907d88b2b980, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1787826268.440961:0:66573:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=open 00000002:00010000:1.0:1787826268.440979:0:66573:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1ff:0x0], intent: open flags 0304001 00010000:00010000:1.0:1787826268.441055:0:66573:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b826 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 66573 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.441065:0:66573:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b826 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66573 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826268.441080:0:66573:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826268.441097:0:66573:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a21c0 x1874670011418112/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826268.441108:0:66573:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b826 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66573 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826268.441137:0:66573:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66573:1874670011418112:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826268.443366:0:66573:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe8c00 going to sleep for 13 seconds 00000100:00100000:1.0:1787826268.450650:0:66573:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66573:1874670011418112:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826268.450683:0:66573:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b826 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66573 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.450701:0:66573:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b826 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66573 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.450714:0:66573:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b826 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66573 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.450722:0:66573:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b826 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66573 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.450731:0:66573:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b826 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66573 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.450739:0:66573:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0a400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826268.450744:0:66573:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b826 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66573 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.450752:0:66573:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b826 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66573 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.450761:0:66573:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b826 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 66573 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.450768:0:66573:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826268.450778:0:66573:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b826 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66573 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.450785:0:66573:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b826 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66573 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826268.450808:0:66573:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908a21c0 x1874670011418112/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826286 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826268.450825:0:66573:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1ff:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826268.451711:0:66573:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826268.451739:0:66573:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826268.451760:0:66573:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00010000:00010000:1.0:1787826268.451776:0:66573:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00000080:00200000:1.0:1787826268.451785:0:66573:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826268.451793:0:66573:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826268.451798:0:66573:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:1.0:1787826268.451805:0:66573:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141de3bf8 00010000:00010000:1.0:1787826268.451809:0:66573:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00010000:00010000:1.0:1787826268.451818:0:66573:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00000080:00200000:1.0:1787826268.451835:0:66573:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826268.451845:0:66573:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=267 00000080:00200000:1.0:1787826268.451854:0:66573:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 44 00000080:00200008:1.0:1787826268.451863:0:66573:0:(file.c:1090:ll_file_open()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00000080:00200080:1.0:1787826268.451908:0:66573:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8) cmd=c1406916 arg=723a20 00000100:00100000:1.0:1787826268.451984:0:66573:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66573:1874670011418240:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826268.452260:0:66573:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe8c00 going to sleep for 13 seconds 00000100:00100000:3.0:1787826268.463235:0:66573:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66573:1874670011418240:192.168.203.104@tcp:34:lfs.0 00000080:00200008:3.0:1787826268.463477:0:66573:0:(file.c:479:ll_file_release()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00010000:00010000:3.0:1787826268.463528:0:66573:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fb01 expref: -99 pid: 66528 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200008:1.0:1787826268.496518:0:66576:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1787826268.496533:0:66576:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826268.496564:0:66576:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826268.496590:0:66576:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00010000:00010000:1.0:1787826268.497183:0:66576:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00000080:00200000:1.0:1787826268.497196:0:66576:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826268.497208:0:66576:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826268.497212:0:66576:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:1.0:1787826268.497222:0:66576:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141dabcb8 00010000:00010000:1.0:1787826268.497227:0:66576:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00010000:00010000:1.0:1787826268.497238:0:66576:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00000080:00200000:1.0:1787826268.497756:0:66576:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826268.498182:0:66576:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826268.498204:0:66576:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00010000:00010000:0.0:1787826268.498220:0:66576:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00000080:00200000:0.0:1787826268.498228:0:66576:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826268.498236:0:66576:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826268.498243:0:66576:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826268.498250:0:66576:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141dabbb0 00010000:00010000:0.0:1787826268.498253:0:66576:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00010000:00010000:0.0:1787826268.498262:0:66576:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 43 lru_type: 1 00000080:00200000:0.0:1787826268.498280:0:66576:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:0.0:1787826268.498372:0:66576:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:0.0:1787826268.498410:0:66576:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffdb6419a5c 00000080:00200008:0.0:1787826268.498429:0:66576:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:2.0:1787826268.505337:0:66578:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826268.505376:0:66578:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826268.505404:0:66578:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 43 lru_type: 1 00010000:00010000:2.0:1787826268.505423:0:66578:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 43 lru_type: 1 00000080:00200000:2.0:1787826268.505433:0:66578:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826268.505445:0:66578:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826268.505449:0:66578:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:2.0:1787826268.505459:0:66578:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141e3fbf8 00010000:00010000:2.0:1787826268.505464:0:66578:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 43 lru_type: 1 00010000:00010000:2.0:1787826268.505479:0:66578:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 44 lru_type: 1 00000080:00200000:2.0:1787826268.505500:0:66578:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826268.505522:0:66578:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=27 00000080:00200000:2.0:1787826268.505530:0:66578:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:2.0:1787826268.505545:0:66578:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), file ffff907d99bc2600, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1787826268.505553:0:66578:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=open 00000002:00010000:2.0:1787826268.505575:0:66578:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1ff:0x0], intent: open flags 0304001 00010000:00010000:2.0:1787826268.505666:0:66578:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa800/0xe5f736940357b82d lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 66578 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826268.505675:0:66578:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa800/0xe5f736940357b82d lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66578 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826268.505690:0:66578:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826268.505708:0:66578:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d91148f00 x1874670011418496/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826268.505718:0:66578:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa800/0xe5f736940357b82d lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66578 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826268.505745:0:66578:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66578:1874670011418496:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826268.506014:0:66578:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb6c0 going to sleep for 13 seconds 00000100:00100000:3.0:1787826268.513057:0:66578:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66578:1874670011418496:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826268.513102:0:66578:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa800/0xe5f736940357b82d lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66578 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.513121:0:66578:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa800/0xe5f736940357b82d lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66578 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.513129:0:66578:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa800/0xe5f736940357b82d lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66578 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.513138:0:66578:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa800/0xe5f736940357b82d lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66578 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.513149:0:66578:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa800/0xe5f736940357b82d lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66578 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.513156:0:66578:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d812aa800 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826268.513161:0:66578:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa800/0xe5f736940357b82d lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66578 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.513169:0:66578:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa800/0xe5f736940357b82d lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66578 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.513177:0:66578:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa800/0xe5f736940357b82d lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 66578 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.513185:0:66578:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826268.513194:0:66578:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa800/0xe5f736940357b82d lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66578 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.513202:0:66578:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa800/0xe5f736940357b82d lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66578 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826268.513227:0:66578:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d91148f00 x1874670011418496/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826286 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826268.513244:0:66578:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1ff:0x0] suppgids 0 -1: rc -20 00000080:00200000:3.0:1787826268.513763:0:66578:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826268.513793:0:66578:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826268.513814:0:66578:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 44 lru_type: 1 00010000:00010000:3.0:1787826268.513829:0:66578:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 44 lru_type: 1 00000080:00200000:3.0:1787826268.513837:0:66578:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826268.513843:0:66578:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826268.513847:0:66578:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:3.0:1787826268.513853:0:66578:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141e3fbf8 00010000:00010000:3.0:1787826268.513857:0:66578:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 44 lru_type: 1 00010000:00010000:3.0:1787826268.513867:0:66578:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 45 lru_type: 1 00000080:00200000:3.0:1787826268.513887:0:66578:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826268.513898:0:66578:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=267 00000080:00200000:3.0:1787826268.513906:0:66578:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 44 00000080:00200008:3.0:1787826268.513917:0:66578:0:(file.c:1090:ll_file_open()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00000080:00200080:3.0:1787826268.513963:0:66578:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8) cmd=c1406916 arg=1612a20 00000100:00100000:3.0:1787826268.514038:0:66578:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66578:1874670011418624:192.168.203.104@tcp:34:lfs.0 00000100:00100000:3.0:1787826268.514412:0:66578:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7db40 going to sleep for 13 seconds 00000100:00100000:3.0:1787826268.525357:0:66578:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66578:1874670011418624:192.168.203.104@tcp:34:lfs.0 00000080:00200008:3.0:1787826268.525585:0:66578:0:(file.c:479:ll_file_release()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00010000:00010000:3.0:1787826268.525664:0:66578:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fb01 expref: -99 pid: 66528 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200008:3.0:1787826268.555499:0:66583:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1787826268.555527:0:66583:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826268.555553:0:66583:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826268.555575:0:66583:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 45 lru_type: 1 00010000:00010000:3.0:1787826268.555594:0:66583:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 45 lru_type: 1 00000080:00200000:3.0:1787826268.555603:0:66583:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826268.555652:0:66583:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826268.555656:0:66583:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:3.0:1787826268.555666:0:66583:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141e67cb8 00010000:00010000:3.0:1787826268.555671:0:66583:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 45 lru_type: 1 00010000:00010000:3.0:1787826268.555681:0:66583:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 46 lru_type: 1 00000080:00200000:3.0:1787826268.555816:0:66583:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826268.555824:0:66583:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826268.555833:0:66583:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 46 lru_type: 1 00010000:00010000:3.0:1787826268.555847:0:66583:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 46 lru_type: 1 00000080:00200000:3.0:1787826268.555854:0:66583:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826268.555858:0:66583:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826268.555860:0:66583:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:3.0:1787826268.555865:0:66583:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141e67bb0 00010000:00010000:3.0:1787826268.555868:0:66583:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 46 lru_type: 1 00010000:00010000:3.0:1787826268.555876:0:66583:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 47 lru_type: 1 00000080:00200000:3.0:1787826268.555886:0:66583:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:3.0:1787826268.555894:0:66583:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:3.0:1787826268.555914:0:66583:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffc55c384bc 00000080:00200008:3.0:1787826268.555929:0:66583:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:0.0:1787826268.557750:0:66586:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826268.557778:0:66586:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826268.557795:0:66586:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 47 lru_type: 1 00010000:00010000:0.0:1787826268.557811:0:66586:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 47 lru_type: 1 00000080:00200000:0.0:1787826268.557819:0:66586:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826268.557827:0:66586:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826268.557831:0:66586:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826268.557839:0:66586:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141ce3bf8 00010000:00010000:0.0:1787826268.557843:0:66586:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 47 lru_type: 1 00010000:00010000:0.0:1787826268.557852:0:66586:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 48 lru_type: 1 00000080:00200000:0.0:1787826268.557868:0:66586:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826268.557877:0:66586:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=27 00000080:00200000:0.0:1787826268.557883:0:66586:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:0.0:1787826268.557897:0:66586:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), file ffff907d83962180, open_flags 18800, mode 0 opened 0 00000080:00200000:0.0:1787826268.557904:0:66586:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=open 00000002:00010000:0.0:1787826268.557922:0:66586:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1ff:0x0], intent: open flags 0304001 00010000:00010000:0.0:1787826268.557989:0:66586:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b834 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 66586 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826268.557998:0:66586:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b834 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66586 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826268.558011:0:66586:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826268.558025:0:66586:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90845680 x1874670011418880/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826268.558036:0:66586:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b834 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66586 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826268.558062:0:66586:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66586:1874670011418880:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826268.558336:0:66586:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd000 going to sleep for 13 seconds 00000100:00100000:1.0:1787826268.566033:0:66586:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66586:1874670011418880:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826268.566066:0:66586:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b834 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66586 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.566081:0:66586:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b834 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66586 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.566088:0:66586:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b834 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66586 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.566095:0:66586:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b834 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66586 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.566109:0:66586:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b834 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66586 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.566116:0:66586:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883b200 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826268.566121:0:66586:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b834 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66586 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.566128:0:66586:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b834 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66586 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.566136:0:66586:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b834 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 66586 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.566145:0:66586:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826268.566153:0:66586:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b834 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66586 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.566160:0:66586:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b834 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66586 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826268.566188:0:66586:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d90845680 x1874670011418880/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826286 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826268.566204:0:66586:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1ff:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826268.566913:0:66586:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826268.566940:0:66586:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826268.566960:0:66586:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 48 lru_type: 1 00010000:00010000:1.0:1787826268.566977:0:66586:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 48 lru_type: 1 00000080:00200000:1.0:1787826268.566985:0:66586:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826268.566992:0:66586:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826268.566996:0:66586:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:1.0:1787826268.567003:0:66586:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141ce3bf8 00010000:00010000:1.0:1787826268.567006:0:66586:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 48 lru_type: 1 00010000:00010000:1.0:1787826268.567015:0:66586:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 49 lru_type: 1 00000080:00200000:1.0:1787826268.567032:0:66586:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826268.567041:0:66586:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=267 00000080:00200000:1.0:1787826268.567049:0:66586:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 44 00000080:00200008:1.0:1787826268.567059:0:66586:0:(file.c:1090:ll_file_open()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00000080:00200080:1.0:1787826268.567104:0:66586:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8) cmd=c1406916 arg=1a03a20 00000100:00100000:1.0:1787826268.567182:0:66586:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66586:1874670011419008:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826268.567449:0:66586:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe80c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826268.582462:0:66586:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66586:1874670011419008:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826268.582572:0:66586:0:(file.c:479:ll_file_release()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00010000:00010000:1.0:1787826268.583177:0:66586:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fb01 expref: -99 pid: 66528 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200008:0.0:1787826268.624450:0:66590:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1787826268.624464:0:66590:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826268.624491:0:66590:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826268.624527:0:66590:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 49 lru_type: 1 00010000:00010000:0.0:1787826268.624547:0:66590:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 49 lru_type: 1 00000080:00200000:0.0:1787826268.624556:0:66590:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826268.624567:0:66590:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826268.624572:0:66590:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826268.624582:0:66590:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141f5bcb8 00010000:00010000:0.0:1787826268.624586:0:66590:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 49 lru_type: 1 00010000:00010000:0.0:1787826268.624596:0:66590:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 50 lru_type: 1 00000080:00200000:0.0:1787826268.625012:0:66590:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826268.625024:0:66590:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826268.625034:0:66590:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 50 lru_type: 1 00010000:00010000:0.0:1787826268.625044:0:66590:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 50 lru_type: 1 00000080:00200000:0.0:1787826268.625052:0:66590:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826268.625057:0:66590:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826268.625059:0:66590:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826268.625064:0:66590:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141f5bbb0 00010000:00010000:0.0:1787826268.625067:0:66590:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 50 lru_type: 1 00010000:00010000:0.0:1787826268.625154:0:66590:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 51 lru_type: 1 00000080:00200000:0.0:1787826268.625165:0:66590:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:0.0:1787826268.625175:0:66590:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:0.0:1787826268.625197:0:66590:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffed505f99c 00000080:00200008:0.0:1787826268.625212:0:66590:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:3.0:1787826268.628818:0:66594:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826268.628860:0:66594:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826268.628885:0:66594:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 51 lru_type: 1 00010000:00010000:3.0:1787826268.628905:0:66594:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 51 lru_type: 1 00000080:00200000:3.0:1787826268.628915:0:66594:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826268.628927:0:66594:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826268.628933:0:66594:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:3.0:1787826268.628942:0:66594:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141fd3bf8 00010000:00010000:3.0:1787826268.628947:0:66594:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 51 lru_type: 1 00010000:00010000:3.0:1787826268.628957:0:66594:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 52 lru_type: 1 00000080:00200000:3.0:1787826268.628975:0:66594:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826268.628984:0:66594:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=27 00000080:00200000:3.0:1787826268.628991:0:66594:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:3.0:1787826268.629004:0:66594:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), file ffff907d9869d980, open_flags 18800, mode 0 opened 0 00000080:00200000:3.0:1787826268.629011:0:66594:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=open 00000002:00010000:3.0:1787826268.629028:0:66594:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1ff:0x0], intent: open flags 0304001 00010000:00010000:3.0:1787826268.629110:0:66594:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eec00/0xe5f736940357b83b lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 66594 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826268.629120:0:66594:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eec00/0xe5f736940357b83b lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66594 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826268.629134:0:66594:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826268.629150:0:66594:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83524780 x1874670011419264/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826268.629161:0:66594:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eec00/0xe5f736940357b83b lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66594 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826268.629187:0:66594:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66594:1874670011419264:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826268.631247:0:66594:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7db40 going to sleep for 13 seconds 00000100:00100000:1.0:1787826268.638405:0:66594:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66594:1874670011419264:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826268.638440:0:66594:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eec00/0xe5f736940357b83b lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66594 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.638458:0:66594:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eec00/0xe5f736940357b83b lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66594 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.638466:0:66594:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eec00/0xe5f736940357b83b lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66594 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.638474:0:66594:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eec00/0xe5f736940357b83b lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66594 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.638484:0:66594:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eec00/0xe5f736940357b83b lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66594 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.638491:0:66594:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907da59eec00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826268.638496:0:66594:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eec00/0xe5f736940357b83b lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66594 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.638518:0:66594:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eec00/0xe5f736940357b83b lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66594 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.638528:0:66594:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eec00/0xe5f736940357b83b lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 66594 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.638536:0:66594:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826268.638545:0:66594:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eec00/0xe5f736940357b83b lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66594 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.638553:0:66594:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59eec00/0xe5f736940357b83b lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66594 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826268.638578:0:66594:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d83524780 x1874670011419264/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826286 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826268.638596:0:66594:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1ff:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826268.638932:0:66594:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826268.638956:0:66594:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826268.638975:0:66594:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 52 lru_type: 1 00010000:00010000:1.0:1787826268.638989:0:66594:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 52 lru_type: 1 00000080:00200000:1.0:1787826268.638996:0:66594:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826268.639003:0:66594:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826268.639007:0:66594:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:1.0:1787826268.639015:0:66594:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141fd3bf8 00010000:00010000:1.0:1787826268.639018:0:66594:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 52 lru_type: 1 00010000:00010000:1.0:1787826268.639027:0:66594:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 53 lru_type: 1 00000080:00200000:1.0:1787826268.639044:0:66594:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826268.639053:0:66594:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=267 00000080:00200000:1.0:1787826268.639062:0:66594:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 44 00000080:00200008:1.0:1787826268.639071:0:66594:0:(file.c:1090:ll_file_open()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00000080:00200080:1.0:1787826268.639364:0:66594:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8) cmd=c1406916 arg=1faaa20 00000100:00100000:1.0:1787826268.639447:0:66594:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66594:1874670011419392:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826268.641810:0:66594:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe80c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826268.646836:0:66594:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66594:1874670011419392:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826268.647209:0:66594:0:(file.c:479:ll_file_release()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00010000:00010000:1.0:1787826268.647249:0:66594:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fb01 expref: -99 pid: 66528 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1787826268.671534:0:66596:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826268.671563:0:66596:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826268.671582:0:66596:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 53 lru_type: 1 00010000:00010000:1.0:1787826268.671598:0:66596:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 53 lru_type: 1 00000080:00200000:1.0:1787826268.671685:0:66596:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826268.671694:0:66596:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826268.671957:0:66596:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:1.0:1787826268.671967:0:66596:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141fd3bf8 00010000:00010000:1.0:1787826268.671972:0:66596:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 53 lru_type: 1 00010000:00010000:1.0:1787826268.671981:0:66596:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 54 lru_type: 1 00000080:00200000:1.0:1787826268.671997:0:66596:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826268.672005:0:66596:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=27 00000080:00200000:1.0:1787826268.672011:0:66596:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:1.0:1787826268.672025:0:66596:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), file ffff907d88b2b980, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1787826268.672032:0:66596:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=open 00000002:00010000:1.0:1787826268.672048:0:66596:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1ff:0x0], intent: open flags 0304001 00010000:00010000:1.0:1787826268.672103:0:66596:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b842 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 66596 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.672184:0:66596:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b842 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66596 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826268.672198:0:66596:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826268.672209:0:66596:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0000 x1874670011419648/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826268.672219:0:66596:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b842 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66596 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826268.672244:0:66596:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66596:1874670011419648:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826268.672473:0:66596:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe8180 going to sleep for 13 seconds 00000100:00100000:1.0:1787826268.684262:0:66596:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66596:1874670011419648:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826268.684286:0:66596:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b842 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 66596 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.684300:0:66596:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b842 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66596 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.684307:0:66596:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b842 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66596 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.684313:0:66596:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b842 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 66596 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.684321:0:66596:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b842 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 66596 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.684328:0:66596:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0a400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826268.684333:0:66596:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b842 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66596 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.684339:0:66596:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b842 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 66596 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.684345:0:66596:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b842 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 66596 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.684351:0:66596:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826268.684475:0:66596:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b842 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66596 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826268.684483:0:66596:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b842 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 66596 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826268.684509:0:66596:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908a0000 x1874670011419648/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826286 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826268.684523:0:66596:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x1ff:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826268.685491:0:66596:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826268.685527:0:66596:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826268.685542:0:66596:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 54 lru_type: 1 00010000:00010000:1.0:1787826268.685556:0:66596:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 54 lru_type: 1 00000080:00200000:1.0:1787826268.685563:0:66596:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826268.685570:0:66596:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826268.685573:0:66596:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:1.0:1787826268.685579:0:66596:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141fd3bf8 00010000:00010000:1.0:1787826268.685583:0:66596:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 54 lru_type: 1 00010000:00010000:1.0:1787826268.685591:0:66596:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 55 lru_type: 1 00000080:00200000:1.0:1787826268.685604:0:66596:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826268.685634:0:66596:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=267 00000080:00200000:1.0:1787826268.685643:0:66596:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 44 00000080:00200008:1.0:1787826268.685653:0:66596:0:(file.c:1090:ll_file_open()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00000080:00200080:1.0:1787826268.685691:0:66596:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8) cmd=c0086915 arg=d963d0 00000100:00100000:1.0:1787826268.687290:0:66596:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66596:1874670011419776:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826268.687399:0:66596:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88fe8180 going to sleep for 13 seconds 00000100:00100000:1.0:1787826268.692377:0:66596:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:66596:1874670011419776:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826268.692564:0:66596:0:(file.c:479:ll_file_release()) START file d204b.sanity-flr:[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags 304000 00010000:00010000:1.0:1787826268.692594:0:66596:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a600/0xe5f736940357b793 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fb01 expref: -99 pid: 66528 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:0.0:1787826269.047432:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826270.069155:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826271.101015:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826271.989142:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011420032:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826271.989251:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011420288:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826271.989287:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1787826271.989293:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2348:1874670011419904:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826271.989319:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011420160:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826271.997117:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011420160:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826271.997134:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011420032:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826271.997145:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011420288:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826272.001022:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2348:1874670011419904:192.168.203.104@tcp:400:kworker.0 00010000:00010000:1.0:1787826272.117176:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826273.141830:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1787826273.658299:0:67014:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826273.658414:0:67014:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826273.658442:0:67014:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 55 lru_type: 1 00010000:00010000:2.0:1787826273.658476:0:67014:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 55 lru_type: 1 00000080:00200000:2.0:1787826273.658487:0:67014:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826273.658498:0:67014:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826273.658503:0:67014:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:2.0:1787826273.658513:0:67014:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141f8bb58 00010000:00010000:2.0:1787826273.658518:0:67014:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 55 lru_type: 1 00010000:00010000:2.0:1787826273.658529:0:67014:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 56 lru_type: 1 00000080:00200000:2.0:1787826273.658553:0:67014:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826273.658563:0:67014:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=24 00000080:00200000:2.0:1787826273.658571:0:67014:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:2.0:1787826273.658587:0:67014:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), flags=8 00000080:00200000:2.0:1787826273.658592:0:67014:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), intent=getattr 00000002:00010000:2.0:1787826273.658637:0:67014:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x1ff:0x0], intent: getattr flags 01 00010000:00010000:2.0:1787826273.658705:0:67014:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 67014 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826273.658715:0:67014:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67014 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826273.658866:0:67014:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67014 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826273.658903:0:67014:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:67014:1874670011420416:192.168.203.104@tcp:101:rm.0 00000100:00100000:2.0:1787826273.659216:0:67014:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fde40 going to sleep for 13 seconds 00000100:00100000:0.0:1787826273.664022:0:67014:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:67014:1874670011420416:192.168.203.104@tcp:101:rm.0 00010000:00010000:0.0:1787826273.664059:0:67014:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x1ff:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fde0 expref: -99 pid: 67014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826273.664153:0:67014:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fde0 expref: -99 pid: 67014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826273.664164:0:67014:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fde0 expref: -99 pid: 67014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826273.664186:0:67014:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fde0 expref: -99 pid: 67014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826273.664201:0:67014:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fde0 expref: -99 pid: 67014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826273.664217:0:67014:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fde0 expref: -99 pid: 67014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826273.664226:0:67014:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fde0 expref: -99 pid: 67014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826273.664235:0:67014:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fde0 expref: -99 pid: 67014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826273.664250:0:67014:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d91149680 x1874670011420416/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/1136 e 0 to 0 dl 1787826291 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826273.664265:0:67014:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fde0 expref: -99 pid: 67014 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1787826273.664282:0:67014:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fde0 expref: -99 pid: 67014 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826273.664294:0:67014:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/512 (0 0) 00000080:00200000:0.0:1787826273.664304:0:67014:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1ff:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1787826273.664383:0:67014:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x200:0x0], updating i_size 2097152 00000080:00200000:0.0:1787826273.664389:0:67014:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x200:0x0](ffff907d93313248): rc = 0 00000080:00200000:0.0:1787826273.664409:0:67014:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x200:0x0]: layout version change: 8 -> 8 00000080:00010000:0.0:1787826273.664415:0:67014:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) for lock 0xe5f736940357b849 00000080:00010000:0.0:1787826273.664431:0:67014:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) 00000080:00010000:0.0:1787826273.664435:0:67014:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b849 from it ffff9dd141f8bb20 00010000:00010000:0.0:1787826273.664439:0:67014:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fde0 expref: -99 pid: 67014 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826273.664449:0:67014:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fde0 expref: -99 pid: 67014 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:0.0:1787826273.664517:0:67014:0:(file.c:6313:ll_getattr_dentry()) START file f204b.sanity-flr:[0x200000404:0x200:0x0](ffff907d93313248), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1787826273.664525:0:67014:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248),name=f204b.sanity-flr 00000002:00010000:0.0:1787826273.664554:0:67014:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x200:0x0]) in obj [0x200000404:0x1ff:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826273.664565:0:67014:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fde0 expref: -99 pid: 67014 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826273.664577:0:67014:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fde0 expref: -99 pid: 67014 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826273.664586:0:67014:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x1ff:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826273.664593:0:67014:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) 00000080:00010000:0.0:1787826273.664596:0:67014:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x200:0x0](ffff907d93313248) for lock 0xe5f736940357b849 00000080:00010000:0.0:1787826273.664600:0:67014:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b849 from it ffff9dd141f8bcb8 00010000:00010000:0.0:1787826273.664603:0:67014:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fde0 expref: -99 pid: 67014 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826273.664638:0:67014:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fde0 expref: -99 pid: 67014 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826273.664711:0:67014:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826273.664721:0:67014:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826273.664730:0:67014:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 56 lru_type: 1 00010000:00010000:0.0:1787826273.664739:0:67014:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 56 lru_type: 1 00000080:00200000:0.0:1787826273.664745:0:67014:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826273.664749:0:67014:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826273.664752:0:67014:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826273.664756:0:67014:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141f8bc10 00010000:00010000:0.0:1787826273.664759:0:67014:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 56 lru_type: 1 00010000:00010000:0.0:1787826273.664766:0:67014:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 57 lru_type: 1 00000080:00200000:0.0:1787826273.664775:0:67014:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826273.664783:0:67014:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=24 00000080:00200000:0.0:1787826273.664789:0:67014:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 1 00000080:00200000:0.0:1787826273.664796:0:67014:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204b.sanity-flr, flags=0 00000080:00200000:0.0:1787826273.664800:0:67014:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x1ff:0x0](ffff907d9221e3c8), inode mode 41ed mask 3 00000080:00200000:0.0:1787826273.664805:0:67014:0:(namei.c:2418:ll_unlink()) VFS Op:name=f204b.sanity-flr, dir=[0x200000404:0x1ff:0x0](ffff907d9221e3c8) 00010000:00010000:0.0:1787826273.664837:0:67014:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0fc74 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1787826273.664858:0:67014:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x200:0x0]: losing layout lock 00010000:00010000:0.0:1787826273.664872:0:67014:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0fde0 expref: -99 pid: 67014 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826273.664880:0:67014:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x200:0x0]: losing layout lock 00010000:00010000:0.0:1787826273.664887:0:67014:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b777 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x200:0x0].0x0 bits 0x20/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc0fa98 expref: -99 pid: 66521 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826273.664937:0:67014:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 1/0,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0fc74 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826273.664946:0:67014:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0fde0 expref: -99 pid: 67014 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826273.664952:0:67014:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b777 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0fa98 expref: -99 pid: 66521 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826273.664959:0:67014:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:0.0:1787826273.664962:0:67014:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812abc00/0xe5f736940357b80a lrc: 0/0,0 mode: --/CR res: [0x200000404:0x200:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0fc74 expref: -99 pid: 66551 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826273.664978:0:67014:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d812aa600/0xe5f736940357b849 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0fde0 expref: -99 pid: 67014 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826273.664988:0:67014:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357b777 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x200:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0fa98 expref: -99 pid: 66521 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000020:00100000:0.0:1787826273.665011:0:67014:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1787826273.665022:0:67014:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908443c0 x1874670011420672/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 512/2200 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787826273.665052:0:67014:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:67014:1874670011420672:192.168.203.104@tcp:36:rm.0 00000100:00100000:0.0:1787826273.666385:0:67014:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d91f56540 going to sleep for 13 seconds 00000100:00100000:0.0:1787826273.676210:0:67014:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:67014:1874670011420672:192.168.203.104@tcp:36:rm.0 00000080:00200000:0.0:1787826273.676369:0:67014:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826273.676386:0:67014:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x200:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826273.676390:0:67014:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826273.676399:0:67014:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826273.676414:0:67014:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:0.0:1787826273.676426:0:67014:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:0.0:1787826273.676455:0:67014:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x200:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1787826273.676464:0:67014:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200000404:0x200:0x0](ffff907d93313248) 00000100:00100000:2.0:1787826273.690998:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826273.695461:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025488512 00000100:00100000:3.0:1787826273.695493:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025488512 00000100:00100000:3.0:1787826273.695569:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7141 00000100:00100000:3.0:1787826273.695581:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91149a40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:28813:x1874670025488512:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826273.695627:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357b7ee lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x309:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc0fc4a expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826273.695752:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826273.695760:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91149a40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:28813:x1874670025488512:12345-192.168.203.104@tcp:104: Request processed in 181us (4773us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826273.695770:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7141 00010000:00010000:1.0:1787826273.696322:0:40346:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826273.696333:0:40346:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357b7ee lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x309:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc0fc4a expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826273.696345:0:40346:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0bc00 already unused, calling callback (ffffffffc118e410) 00010000:00010000:1.0:1787826273.696353:0:40346:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357b7ee lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x309:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x428400010000 nid: local remote: 0x7918213f4fc0fc4a expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826273.696418:0:40346:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357b7ee lrc: 2/0,0 mode: --/PW res: [0x280000400:0x309:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc0fc4a expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826273.696428:0:40346:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826273.696458:0:40346:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357b7ee lrc: 1/0,0 mode: --/PW res: [0x280000400:0x309:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc0fc4a expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826273.696477:0:40346:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357b7ee lrc: 0/0,0 mode: --/PW res: [0x280000400:0x309:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc0fc4a expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826273.696504:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826273.696506:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826273.696508:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826273.696529:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011420800:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1787826273.714598:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826273.714709:0:40338:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:3.0:1787826273.714733:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025488640 00000100:00100000:3.0:1787826273.714743:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025488640 00000100:00100000:3.0:1787826273.714806:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7142 00000100:00100000:3.0:1787826273.714818:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d88634850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:36729:x1874670025488640:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826273.714844:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x420000030000 nid: local remote: 0x7918213f4fc0fc5f expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826273.714935:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826273.714942:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d88634850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:36729:x1874670025488640:12345-192.168.203.104@tcp:104: Request processed in 127us (363us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826273.714950:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7142 00010000:00010000:3.0:1787826273.714992:0:60482:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1787826273.715000:0:60482:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x420000030000 nid: local remote: 0x7918213f4fc0fc5f expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826273.715016:0:60482:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883ba00 already unused, calling callback (ffffffffc118e410) 00010000:00010000:3.0:1787826273.715023:0:60482:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x428400030000 nid: local remote: 0x7918213f4fc0fc5f expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826273.715821:0:60482:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 2/0,0 mode: --/PW res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc0fc5f expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826273.715834:0:60482:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787826273.715861:0:60482:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 1/0,0 mode: --/PW res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc0fc5f expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826273.715870:0:60482:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b7fc lrc: 0/0,0 mode: --/PW res: [0x240000400:0x324:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc0fc5f expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826273.715893:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826273.715895:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826273.715896:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826273.715914:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011420928:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826273.716433:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011420800:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1787826273.722638:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011420928:192.168.203.104@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1787826274.167695:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826275.196362:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1787826275.240296:0:67153:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-flr test 204c: FLR write/stale/resync test with component removal ========================================================== 06:24:33 (1787826273) 00000080:00200008:0.0:1787826275.479925:0:67171:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1787826275.479935:0:67171:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826275.479955:0:67171:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826275.479974:0:67171:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 57 lru_type: 1 00010000:00010000:0.0:1787826275.479989:0:67171:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 57 lru_type: 1 00000080:00200000:0.0:1787826275.479997:0:67171:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826275.480004:0:67171:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826275.480008:0:67171:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826275.480016:0:67171:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141853cb8 00010000:00010000:0.0:1787826275.480020:0:67171:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 57 lru_type: 1 00010000:00010000:0.0:1787826275.480028:0:67171:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 58 lru_type: 1 00000080:00200000:0.0:1787826275.480111:0:67171:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826275.480119:0:67171:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826275.480125:0:67171:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 58 lru_type: 1 00010000:00010000:0.0:1787826275.480133:0:67171:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 58 lru_type: 1 00000080:00200000:0.0:1787826275.480140:0:67171:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826275.480144:0:67171:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826275.480146:0:67171:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826275.480150:0:67171:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141853bb0 00010000:00010000:0.0:1787826275.480152:0:67171:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 58 lru_type: 1 00010000:00010000:0.0:1787826275.480160:0:67171:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 59 lru_type: 1 00000080:00200000:0.0:1787826275.480168:0:67171:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:0.0:1787826275.480174:0:67171:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:0.0:1787826275.480190:0:67171:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7fffe4fdd3ec 00000080:00200008:0.0:1787826275.480206:0:67171:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:0.0:1787826275.480226:0:67171:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826275.480234:0:67171:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826275.480240:0:67171:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 59 lru_type: 1 00010000:00010000:0.0:1787826275.481547:0:67171:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 59 lru_type: 1 00000080:00200000:0.0:1787826275.481558:0:67171:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826275.481566:0:67171:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826275.481570:0:67171:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826275.481576:0:67171:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141853bb0 00010000:00010000:0.0:1787826275.481580:0:67171:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 59 lru_type: 1 00010000:00010000:0.0:1787826275.481587:0:67171:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 60 lru_type: 1 00000080:00200000:0.0:1787826275.481601:0:67171:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:0.0:1787826275.481709:0:67171:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 100000 00000080:00200080:0.0:1787826275.481730:0:67171:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=8080667f arg=7fffe4fdc0c0 00000080:00200008:0.0:1787826275.482065:0:67171:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 100000 00000080:00200008:0.0:1787826275.483694:0:67171:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1787826275.483704:0:67171:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826275.483720:0:67171:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826275.483737:0:67171:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 60 lru_type: 1 00010000:00010000:0.0:1787826275.483756:0:67171:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 60 lru_type: 1 00000080:00200000:0.0:1787826275.483763:0:67171:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826275.483771:0:67171:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826275.483774:0:67171:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826275.483780:0:67171:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd141853cb8 00010000:00010000:0.0:1787826275.483784:0:67171:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 60 lru_type: 1 00010000:00010000:0.0:1787826275.483792:0:67171:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 61 lru_type: 1 00000080:00200000:0.0:1787826275.540931:0:67177:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826275.540954:0:67177:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826275.540972:0:67177:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 61 lru_type: 1 00010000:00010000:0.0:1787826275.540987:0:67177:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 61 lru_type: 1 00000080:00200000:0.0:1787826275.540996:0:67177:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826275.541002:0:67177:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826275.541006:0:67177:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826275.541012:0:67177:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd140c8fbe0 00010000:00010000:0.0:1787826275.541016:0:67177:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 61 lru_type: 1 00010000:00010000:0.0:1787826275.541025:0:67177:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 62 lru_type: 1 00000080:00200000:0.0:1787826275.541038:0:67177:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826275.541050:0:67177:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d204c.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), flags=1536 00000080:00200000:0.0:1787826275.541054:0:67177:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826275.541059:0:67177:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826275.541064:0:67177:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 62 lru_type: 1 00010000:00010000:0.0:1787826275.541072:0:67177:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 62 lru_type: 1 00000080:00200000:0.0:1787826275.541078:0:67177:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826275.541081:0:67177:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826275.541084:0:67177:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826275.541087:0:67177:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd140c8fce0 00010000:00010000:0.0:1787826275.541089:0:67177:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 62 lru_type: 1 00010000:00010000:0.0:1787826275.541096:0:67177:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 63 lru_type: 1 00000080:00200000:0.0:1787826275.541103:0:67177:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 3 00000080:00200000:0.0:1787826275.541107:0:67177:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826275.541112:0:67177:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826275.541117:0:67177:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 63 lru_type: 1 00010000:00010000:0.0:1787826275.541124:0:67177:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 63 lru_type: 1 00000080:00200000:0.0:1787826275.541129:0:67177:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826275.541132:0:67177:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826275.541135:0:67177:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b731 00000080:00010000:0.0:1787826275.541137:0:67177:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b731 from it ffff9dd140c8fdc8 00010000:00010000:0.0:1787826275.541140:0:67177:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 63 lru_type: 1 00010000:00010000:0.0:1787826275.541147:0:67177:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 64 lru_type: 1 00000080:00200000:0.0:1787826275.541154:0:67177:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 3 00000080:00200000:0.0:1787826275.541159:0:67177:0:(namei.c:2239:do_mkdir()) VFS Op:name=d204c.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148) 00010000:00010000:0.0:1787826275.541189:0:67177:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 64 lru_type: 1 00010000:00010000:0.0:1787826275.541236:0:67177:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 64 lru_type: 1 00010000:00010000:0.0:1787826275.541381:0:67177:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826275.541384:0:67177:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da639b600/0xe5f736940357b731 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0f9b1 expref: -99 pid: 66520 timeout: 0 lvb_type: 0 lru_score: 64 lru_type: 1 00000020:00100000:0.0:1787826275.541413:0:67177:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1787826275.541425:0:67177:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90847480 x1874670011421184/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 568/2152 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787826275.541449:0:67177:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job mkdir:6f891bcb-15cd-41e0-96e2-db843619c1be:67177:1874670011421184:192.168.203.104@tcp:36:mkdir.0 00000100:00100000:0.0:1787826275.541600:0:67177:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f91a80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826275.551369:0:67177:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589942972 gen 1 00000100:00100000:0.0:1787826275.551398:0:67177:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff907d90847480 x1874670011421184/t8589942973(8589942973) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 568/448 e 0 to 0 dl 1787826293 ref 3 fl Rpc:RQU/204/0 rc 0/0 job:'mkdir.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787826275.551413:0:67177:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job mkdir:6f891bcb-15cd-41e0-96e2-db843619c1be:67177:1874670011421184:192.168.203.104@tcp:36:mkdir.0 00000080:00200000:0.0:1787826275.551467:0:67177:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248) 00000080:00200000:0.0:1787826275.551474:0:67177:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x202:0x0], updating i_size 4096 00000080:00200000:0.0:1787826275.605118:0:67180:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826275.605146:0:67180:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826275.605211:0:67180:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.605221:0:67180:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.605230:0:67180:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826275.605435:0:67180:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67180:1874670011421312:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826275.605563:0:67180:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f91a80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826275.615015:0:67180:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67180:1874670011421312:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826275.615041:0:67180:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.615058:0:67180:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.615069:0:67180:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.615080:0:67180:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.615092:0:67180:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.615100:0:67180:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.615107:0:67180:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826275.615117:0:67180:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d908443c0 x1874670011421312/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/688 e 0 to 0 dl 1787826293 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826275.615130:0:67180:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.615142:0:67180:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1787826275.615148:0:67180:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1787826275.615174:0:67180:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:0.0:1787826275.615180:0:67180:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:0.0:1787826275.615186:0:67180:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826275.615189:0:67180:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd140d53b58 00010000:00010000:0.0:1787826275.615192:0:67180:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.615201:0:67180:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826275.615222:0:67180:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826275.615911:0:67180:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d204c.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), flags=29 00000080:00200000:0.0:1787826275.615919:0:67180:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d204c.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=0 00000002:00010000:0.0:1787826275.615941:0:67180:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: d204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00010000:00010000:0.0:1787826275.615987:0:67180:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b857 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.615996:0:67180:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b857 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.616005:0:67180:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b857 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826275.616027:0:67180:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67180:1874670011421440:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826275.616116:0:67180:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f91a80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826275.625667:0:67180:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67180:1874670011421440:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826275.625693:0:67180:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b857 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe42 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.625722:0:67180:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b857 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe42 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.625731:0:67180:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b857 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x202:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe42 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.625742:0:67180:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b857 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe42 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.625753:0:67180:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b857 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe42 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.625766:0:67180:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b857 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe42 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.625774:0:67180:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b857 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe42 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.625782:0:67180:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b857 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe42 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826275.625794:0:67180:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x10 disposition=0xb, status=0 req@ffff907d908443c0 x1874670011421440/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/584 e 0 to 0 dl 1787826293 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826275.625807:0:67180:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b857 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe42 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.625823:0:67180:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/514 (0 0) 00000080:00200000:0.0:1787826275.625830:0:67180:0:(namei.c:952:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1787826275.625850:0:67180:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x202:0x0], updating i_size 4096 00000080:00200000:0.0:1787826275.625854:0:67180:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x202:0x0](ffff907d93313248): rc = 0 00000080:00010000:0.0:1787826275.625859:0:67180:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x202:0x0](ffff907d93313248) for lock 0xe5f736940357b857 00000080:00010000:0.0:1787826275.625871:0:67180:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x202:0x0](ffff907d93313248) 00000080:00010000:0.0:1787826275.625875:0:67180:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b857 from it ffff9dd140d53a28 00010000:00010000:0.0:1787826275.625879:0:67180:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b857 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe42 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.625887:0:67180:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b857 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe42 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826275.625919:0:67180:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 1 00000080:00200000:0.0:1787826275.625937:0:67180:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), flags=13 00000080:00200000:0.0:1787826275.625942:0:67180:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), intent=getattr 00000002:00010000:0.0:1787826275.625959:0:67180:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x202:0x0], intent: getattr flags 01 00010000:00010000:0.0:1787826275.626000:0:67180:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b85e lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.626007:0:67180:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b85e lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.626014:0:67180:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b85e lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826275.626032:0:67180:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67180:1874670011421568:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826275.626115:0:67180:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f91a80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826275.632302:0:67180:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67180:1874670011421568:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826275.632324:0:67180:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b85e lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.632337:0:67180:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b85e lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.632344:0:67180:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b85e lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.632350:0:67180:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b85e lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.632359:0:67180:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b85e lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.632366:0:67180:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883b400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826275.632370:0:67180:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b85e lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.632376:0:67180:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b85e lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.632382:0:67180:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b85e lrc: 5/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.632390:0:67180:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826275.632396:0:67180:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b85e lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.632408:0:67180:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b85e lrc: 0/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826275.632428:0:67180:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff907d908443c0 x1874670011421568/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/584 e 0 to 0 dl 1787826293 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826275.632442:0:67180:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x202:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1787826275.632476:0:67180:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 30 res 8589935620/514 (0 0) 00000080:00200000:0.0:1787826275.632545:0:67180:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826275.632560:0:67180:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826275.632571:0:67180:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826275.632581:0:67180:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826275.632588:0:67180:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826275.632592:0:67180:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826275.632595:0:67180:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:0.0:1787826275.632600:0:67180:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd140d53be8 00010000:00010000:0.0:1787826275.632602:0:67180:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826275.632665:0:67180:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826275.632677:0:67180:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826275.632685:0:67180:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=16408 00000080:00200000:0.0:1787826275.632690:0:67180:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 1 00000080:00200000:0.0:1787826275.632701:0:67180:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), flags=16392 00000080:00200000:0.0:1787826275.632705:0:67180:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), intent=getattr 00000002:00010000:0.0:1787826275.632726:0:67180:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x202:0x0], intent: getattr flags 01 00010000:00010000:0.0:1787826275.632761:0:67180:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b865 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.632769:0:67180:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b865 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.632777:0:67180:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b865 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826275.632797:0:67180:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67180:1874670011421696:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826275.632893:0:67180:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f91a80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826275.643009:0:67180:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67180:1874670011421696:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826275.643031:0:67180:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b865 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.643045:0:67180:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b865 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.643052:0:67180:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b865 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.643060:0:67180:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b865 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.643068:0:67180:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b865 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.643079:0:67180:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883ba00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826275.643083:0:67180:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b865 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.643090:0:67180:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b865 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.643097:0:67180:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b865 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.643102:0:67180:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826275.643109:0:67180:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b865 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.643115:0:67180:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b865 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826275.643131:0:67180:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff907d908443c0 x1874670011421696/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/584 e 0 to 0 dl 1787826293 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826275.643145:0:67180:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x202:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1787826275.643166:0:67180:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 30 res 8589935620/514 (0 0) 00000080:00200000:0.0:1787826275.643235:0:67180:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826275.643249:0:67180:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826275.643259:0:67180:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826275.643270:0:67180:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826275.643282:0:67180:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826275.643287:0:67180:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826275.643290:0:67180:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:0.0:1787826275.643295:0:67180:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd140d53b58 00010000:00010000:0.0:1787826275.643298:0:67180:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826275.643306:0:67180:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1787826275.643319:0:67180:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826275.643326:0:67180:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=13 00000080:00200008:0.0:1787826275.643334:0:67180:0:(file.c:6313:ll_getattr_dentry()) START file d204c.sanity-flr:[0x200000404:0x202:0x0](ffff907d93313248), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1787826275.643339:0:67180:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248),name=d204c.sanity-flr 00000002:00010000:0.0:1787826275.643349:0:67180:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x202:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826275.643354:0:67180:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 30 res 8589935620/514 (0 0) 00010000:00010000:0.0:1787826275.643388:0:67180:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.643395:0:67180:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.643404:0:67180:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826275.643425:0:67180:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67180:1874670011421824:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826275.643517:0:67180:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f91a80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826275.648526:0:67180:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67180:1874670011421824:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826275.648555:0:67180:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe7a expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.649078:0:67180:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 4/1,0 mode: --/PR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe7a expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.649087:0:67180:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 4/1,0 mode: --/PR res: [0x200000404:0x202:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe7a expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.649098:0:67180:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe7a expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.649110:0:67180:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe7a expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.649125:0:67180:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe7a expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.649134:0:67180:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe7a expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.649141:0:67180:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe7a expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826275.649154:0:67180:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d908443c0 x1874670011421824/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/584 e 0 to 0 dl 1787826293 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826275.649167:0:67180:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe7a expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.649178:0:67180:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/514 (0 0) 00000080:00200000:0.0:1787826275.649184:0:67180:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1787826275.649544:0:67180:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x202:0x0], updating i_size 4096 00000080:00010000:0.0:1787826275.649552:0:67180:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x202:0x0](ffff907d93313248) for lock 0xe5f736940357b86c 00000080:00010000:0.0:1787826275.649561:0:67180:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x202:0x0](ffff907d93313248) 00000080:00010000:0.0:1787826275.649564:0:67180:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b86c from it ffff9dd140d53cb8 00010000:00010000:0.0:1787826275.649567:0:67180:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe7a expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.649575:0:67180:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe7a expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:0.0:1787826275.652070:0:67180:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1787826275.652079:0:67180:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826275.652097:0:67180:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826275.652113:0:67180:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826275.652127:0:67180:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1787826275.652134:0:67180:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826275.652142:0:67180:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826275.652145:0:67180:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:0.0:1787826275.652152:0:67180:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd140d53cb8 00010000:00010000:0.0:1787826275.652155:0:67180:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826275.652164:0:67180:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1787826275.652442:0:67180:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826275.652451:0:67180:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826275.652470:0:67180:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1787826275.652481:0:67180:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1787826275.652488:0:67180:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826275.652493:0:67180:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826275.652496:0:67180:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:0.0:1787826275.652500:0:67180:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd140d53b58 00010000:00010000:0.0:1787826275.652504:0:67180:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1787826275.652511:0:67180:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:0.0:1787826275.652520:0:67180:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826275.652526:0:67180:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=24 00000080:00200000:0.0:1787826275.652531:0:67180:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 1 00000080:00200000:0.0:1787826275.652542:0:67180:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), flags=8 00000080:00200000:0.0:1787826275.652546:0:67180:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), intent=getattr 00000002:00010000:0.0:1787826275.652565:0:67180:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x202:0x0], intent: getattr flags 01 00010000:00010000:0.0:1787826275.652768:0:67180:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b873 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.652778:0:67180:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b873 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.652789:0:67180:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b873 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826275.652815:0:67180:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67180:1874670011421952:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826275.652908:0:67180:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f91a80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826275.659357:0:67180:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67180:1874670011421952:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826275.659378:0:67180:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b873 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.659392:0:67180:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b873 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.659398:0:67180:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b873 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.659405:0:67180:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b873 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.659412:0:67180:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b873 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.659418:0:67180:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883ba00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826275.659422:0:67180:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b873 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.659429:0:67180:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b873 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.659437:0:67180:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b873 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x20e400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.659444:0:67180:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826275.659451:0:67180:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b873 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.659467:0:67180:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b873 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826275.659484:0:67180:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff907d908443c0 x1874670011421952/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/584 e 0 to 0 dl 1787826293 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826275.659498:0:67180:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x202:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1787826275.659522:0:67180:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe7a expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826275.659535:0:67180:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe7a expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00010000:0.0:1787826275.659543:0:67180:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b86c from it ffff9dd140d53930 00010000:00010000:0.0:1787826275.659547:0:67180:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe7a expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826275.659554:0:67180:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe7a expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826275.659684:0:67180:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826275.659699:0:67180:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826275.659708:0:67180:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1787826275.659716:0:67180:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:0.0:1787826275.659723:0:67180:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826275.659728:0:67180:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826275.659731:0:67180:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:0.0:1787826275.659736:0:67180:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd140d53bf8 00010000:00010000:0.0:1787826275.659739:0:67180:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1787826275.659745:0:67180:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:0.0:1787826275.659754:0:67180:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826275.659760:0:67180:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=25 00000080:00200000:0.0:1787826275.659765:0:67180:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 1 00000080:00200000:0.0:1787826275.659771:0:67180:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204c.sanity-flr, flags=777 00000080:00200000:0.0:1787826275.659774:0:67180:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 3 00000080:00200000:0.0:1787826275.659778:0:67180:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), file ffff907d98b6b200, open_flags a141, mode 81b6 opened 0 00000080:00200000:0.0:1787826275.659787:0:67180:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), intent=open|creat 00000002:00010000:0.0:1787826275.659806:0:67180:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204c.sanity-flr,[0x200000404:0x203:0x0]) in obj [0x200000404:0x202:0x0], intent: open|creat flags 0120502 00010000:00010000:0.0:1787826275.659817:0:67180:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc0fe7a expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826275.659830:0:67180:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883bc00/0xe5f736940357b857 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x202:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe42 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826275.659864:0:67180:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 1/0,0 mode: --/PR res: [0x200000404:0x202:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0fe7a expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826275.659871:0:67180:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826275.659873:0:67180:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b86c lrc: 0/0,0 mode: --/PR res: [0x200000404:0x202:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0fe7a expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826275.659894:0:67180:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 3/0,1 mode: --/CW res: [0x200000404:0x202:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.659900:0:67180:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 3/0,1 mode: --/CW res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826275.659910:0:67180:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826275.659917:0:67180:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908443c0 x1874670011422208/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/67896 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826275.659926:0:67180:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 3/0,1 mode: --/CW res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826275.659946:0:67180:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67180:1874670011422208:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826275.660077:0:67180:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f91a80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826275.664190:0:67180:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67180:1874670011422208:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826275.664216:0:67180:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 4/0,1 mode: --/CW res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0feab expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.664238:0:67180:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 4/0,1 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0feab expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.664246:0:67180:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 4/0,1 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0feab expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.664258:0:67180:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0feab expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.664269:0:67180:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0feab expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.664283:0:67180:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0feab expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.664292:0:67180:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0feab expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.664299:0:67180:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0feab expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826275.664311:0:67180:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x8000037, status=0 req@ffff907d908443c0 x1874670011422208/t8589942974(8589942974) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826293 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1787826275.664325:0:67180:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908443c0 x1874670011422208/t8589942974(8589942974) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826293 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826275.664337:0:67180:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0feab expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.664348:0:67180:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/515 (0 0) 00000080:00200000:0.0:1787826275.664354:0:67180:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x200000404:0x202:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1787826275.664365:0:67180:0:(namei.c:1628:ll_create_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), intent=open|creat 00000080:00200000:0.0:1787826275.664391:0:67180:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908) 00000080:00200000:0.0:1787826275.664398:0:67180:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x203:0x0], updating i_size 0 00000080:00200000:0.0:1787826275.664431:0:67180:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x203:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:0.0:1787826275.664436:0:67180:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x200000404:0x203:0x0](ffff907d93310908) 00000080:00010000:0.0:1787826275.664439:0:67180:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) for lock 0xe5f736940357b87a 00000080:00010000:0.0:1787826275.664449:0:67180:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) 00000080:00200008:0.0:1787826275.664473:0:67180:0:(file.c:1090:ll_file_open()) START file f204c.sanity-flr:[0x200000404:0x203:0x0](ffff907d93310908), flags 120501 00000002:00100000:0.0:1787826275.664491:0:67180:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908443c0 x1874670011422208/t8589942974(8589942974) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826293 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:0.0:1787826275.664526:0:67180:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b87a from it ffff907d832413c0 00010000:00010000:0.0:1787826275.664531:0:67180:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0feab expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.664540:0:67180:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0feab expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826275.664561:0:67180:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908), inode mode 81a4 mask 40 00000080:00200000:0.0:1787826275.664632:0:67180:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908), inode mode 81a4 mask 2 00000080:00200000:0.0:1787826275.664642:0:67180:0:(xattr.c:414:ll_xattr_set()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908), xattr lov 00000002:00010000:0.0:1787826275.664658:0:67180:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x203:0x0]) in obj [0x200000404:0x202:0x0], intent: open flags 040000002 00010000:00010000:0.0:1787826275.664702:0:67180:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b881 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.664714:0:67180:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b881 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826275.664724:0:67180:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826275.664730:0:67180:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90847840 x1874670011422464/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826275.664740:0:67180:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b881 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826275.664758:0:67180:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67180:1874670011422464:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826275.664916:0:67180:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f91a80 going to sleep for 13 seconds 00000100:00100000:2.0:1787826275.669368:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826275.669483:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025489024 00000100:00100000:3.0:1787826275.669492:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025489024 00000100:00100000:3.0:1787826275.670186:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7143 00000100:00100000:3.0:1787826275.670200:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d911483c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:36493:x1874670025489024:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826275.670220:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0feab expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826275.670296:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826275.670302:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d911483c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:36493:x1874670025489024:12345-192.168.203.104@tcp:104: Request processed in 107us (944us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826275.670310:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7143 00010000:00010000:0.0:1787826275.670369:0:60483:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826275.670377:0:60483:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc0feab expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826275.670385:0:60483:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a000 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826275.670390:0:60483:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc0feab expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826275.670397:0:60483:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc0feab expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826275.670404:0:60483:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 3/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x7918213f4fc0feab expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826275.670428:0:60483:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x203:0x0]: losing layout lock 00010000:00010000:0.0:1787826275.670483:0:60483:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 2/0,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0feab expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826275.670491:0:60483:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826275.670526:0:60483:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 1/0,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0feab expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826275.670534:0:60483:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b87a lrc: 0/0,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc0feab expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826275.670560:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826275.670562:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826275.670563:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826275.670573:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1787826275.670587:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011422592:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826275.674487:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011422592:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826275.677572:0:67180:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67180:1874670011422464:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826275.677597:0:67180:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b881 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.677709:0:67180:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b881 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.677717:0:67180:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b881 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.677723:0:67180:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b881 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.677740:0:67180:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b881 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.677747:0:67180:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883ba00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826275.677751:0:67180:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b881 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.677758:0:67180:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b881 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.677765:0:67180:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b881 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.677772:0:67180:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826275.677779:0:67180:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b881 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826275.677786:0:67180:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b881 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 67180 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826275.678144:0:67180:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x800002b, status=0 req@ffff907d90847840 x1874670011422464/t8589942975(8589942975) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1264 e 0 to 0 dl 1787826293 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1787826275.678157:0:67180:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90847840 x1874670011422464/t8589942975(8589942975) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1264 e 0 to 0 dl 1787826293 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826275.678174:0:67180:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x202:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826275.678316:0:67180:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1787826275.678325:0:67180:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826275.678370:0:67180:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x203:0x0], updating i_size 0 00000002:00100000:1.0:1787826275.678388:0:67180:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90847840 x1874670011422464/t8589942975(8589942975) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1176/1264 e 0 to 0 dl 1787826293 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826275.678423:0:67180:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d90847840 x1874670011422464/t8589942975(8589942975) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1176/1264 e 0 to 0 dl 1787826293 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826275.678437:0:67180:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826275.678443:0:67180:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0b40 x1874670011422848/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826275.678475:0:67180:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67180:1874670011422848:192.168.203.104@tcp:35:lfs.0 00000100:00100000:1.0:1787826275.678653:0:67180:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d96147000 going to sleep for 11 seconds 00000100:00100000:1.0:1787826275.686184:0:67180:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67180:1874670011422848:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826275.686233:0:67180:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d90847840 x1874670011422464/t8589942975(8589942975) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1176/1264 e 0 to 0 dl 1787826293 ref 2 fl Complete:RQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1787826275.686300:0:67180:0:(file.c:479:ll_file_release()) START file f204c.sanity-flr:[0x200000404:0x203:0x0](ffff907d93310908), flags 100001 00000002:00100000:1.0:1787826275.686351:0:67180:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d908443c0 x1874670011422208/t8589942974(8589942974) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826293 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826275.686362:0:67180:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826275.686368:0:67180:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a1a40 x1874670011423104/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826275.686389:0:67180:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67180:1874670011423104:192.168.203.104@tcp:35:lfs.0 00000100:00100000:1.0:1787826275.686475:0:67180:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d96147000 going to sleep for 11 seconds 00000100:00100000:1.0:1787826275.690224:0:67180:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67180:1874670011423104:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826275.690257:0:67180:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d908443c0 x1874670011422208/t8589942974(8589942974) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826293 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:0.0:1787826276.212828:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826277.108932:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011423360:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826277.109150:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011423616:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826277.109176:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011423232:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826277.109209:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011423488:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826277.127041:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011423232:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826277.127068:0:2349:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff907d986d0000: committing for last_committed 21474840496 gen 3 00000100:00100000:1.0:1787826277.127096:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011423488:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826277.127241:0:2348:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0001-osc-ffff907d986d0000: committing for last_committed 17179873832 gen 1 00000100:00100000:0.0:1787826277.127262:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011423616:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826277.134275:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011423360:192.168.203.104@tcp:400:kworker.0 00010000:00010000:1.0:1787826277.240297:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826278.262011:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826279.154269:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826279.154370:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025490048 00000100:00100000:3.0:1787826279.154382:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025490048 00000100:00100000:3.0:1787826279.154463:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7144 00000100:00100000:3.0:1787826279.154475:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83526580 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:35521:x1874670025490048:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826279.154506:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x420000030000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826279.154656:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826279.154664:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83526580 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:35521:x1874670025490048:12345-192.168.203.104@tcp:104: Request processed in 191us (424us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826279.154673:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7144 00010000:00010000:1.0:1787826279.154960:0:40346:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826279.154970:0:40346:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x420000030000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826279.154980:0:40346:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a200 already unused, calling callback (ffffffffc118e410) 00010000:00010000:1.0:1787826279.154988:0:40346:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x428400030000 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826279.155057:0:40346:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826279.155065:0:40346:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826279.155113:0:40346:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826279.155121:0:40346:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b803 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x325:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc0fc66 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826279.155149:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826279.155151:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826279.155153:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826279.155292:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1787826279.155314:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011423744:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1787826279.160082:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826279.160411:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025490304 00000100:00100000:2.0:1787826279.160424:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025490304 00000100:00100000:2.0:1787826279.160509:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7145 00000100:00100000:2.0:1787826279.160522:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83526940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:36513:x1874670025490304:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826279.160548:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357b7f5 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x30a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc0fc51 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826279.160686:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826279.160694:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83526940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:36513:x1874670025490304:12345-192.168.203.104@tcp:104: Request processed in 175us (626us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826279.160702:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7145 00010000:00010000:0.0:1787826279.161602:0:60482:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826279.161661:0:60482:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357b7f5 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x30a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc0fc51 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826279.161675:0:60482:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0b400 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826279.161683:0:60482:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357b7f5 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x30a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x428400010000 nid: local remote: 0x7918213f4fc0fc51 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826279.161750:0:60482:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357b7f5 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x30a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc0fc51 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826279.161759:0:60482:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826279.161787:0:60482:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357b7f5 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x30a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc0fc51 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826279.161796:0:60482:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357b7f5 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x30a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc0fc51 expref: -99 pid: 66551 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826279.161828:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826279.161830:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826279.161831:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826279.161844:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1787826279.161869:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011423872:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826279.170244:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011423872:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826279.172547:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011423744:192.168.203.104@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1787826279.289679:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826280.309270:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826281.332955:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826282.230114:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011424000:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826282.230246:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011424256:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826282.230280:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1787826282.230285:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011424128:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826282.230306:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011424384:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826282.236503:0:2349:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589942977 gen 1 00000100:00100000:1.0:1787826282.236648:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011424128:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826282.236680:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011424384:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826282.236697:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011424256:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826282.239665:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011424000:192.168.203.104@tcp:400:kworker.0 00010000:00010000:0.0:1787826282.356826:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826283.382866:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1787826284.276815:0:37:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f7:0x0] ignore/verify layout 1/0, layout version 6 restore needed 0 00020000:00200000:3.0:1787826284.276831:0:37:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1787826284.276841:0:37:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00000080:00200000:3.0:1787826284.276873:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f7:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1787826284.404997:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826285.428844:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826286.453003:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826287.348846:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011424640:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826287.348962:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011424896:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826287.348999:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1787826287.349006:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2348:1874670011424512:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826287.349033:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011424768:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826287.352109:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2348:1874670011424512:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826287.352132:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011424768:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826287.352142:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011424640:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826287.352152:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011424896:192.168.203.104@tcp:400:kworker.0 00010000:00010000:2.0:1787826287.484078:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1787826287.593327:0:67203:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826287.593355:0:67203:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826287.593385:0:67203:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1787826287.593401:0:67203:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:0.0:1787826287.593414:0:67203:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826287.593421:0:67203:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826287.593425:0:67203:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:0.0:1787826287.593433:0:67203:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141c3fbf8 00010000:00010000:0.0:1787826287.593437:0:67203:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1787826287.593446:0:67203:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:0.0:1787826287.593462:0:67203:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826287.593469:0:67203:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=25 00000080:00200000:0.0:1787826287.593475:0:67203:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 1 00000080:00200000:0.0:1787826287.593485:0:67203:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 3 00000080:00200000:0.0:1787826287.593489:0:67203:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), file ffff907d82cb1680, open_flags 8041, mode 81b6 opened 0 00000080:00200000:0.0:1787826287.593497:0:67203:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), intent=open|creat 00000002:00010000:0.0:1787826287.593515:0:67203:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204c.sanity-flr,[0x200000404:0x204:0x0]) in obj [0x200000404:0x202:0x0], intent: open|creat flags 0100102 00010000:00010000:0.0:1787826287.593566:0:67203:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b888 lrc: 3/0,1 mode: --/CW res: [0x200000404:0x202:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67203 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.593574:0:67203:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b888 lrc: 3/0,1 mode: --/CW res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67203 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826287.593585:0:67203:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826287.593597:0:67203:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908461c0 x1874670011425152/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/67896 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826287.593638:0:67203:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b888 lrc: 3/0,1 mode: --/CW res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67203 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826287.593661:0:67203:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:67203:1874670011425152:192.168.203.104@tcp:101:dd.0 00000100:00100000:0.0:1787826287.593782:0:67203:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f91a80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826287.605531:0:67203:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:67203:1874670011425152:192.168.203.104@tcp:101:dd.0 00010000:00010000:0.0:1787826287.605559:0:67203:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b888 lrc: 4/0,1 mode: --/CW res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fef1 expref: -99 pid: 67203 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.605579:0:67203:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b888 lrc: 4/0,1 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fef1 expref: -99 pid: 67203 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.605588:0:67203:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b888 lrc: 4/0,1 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fef1 expref: -99 pid: 67203 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.605599:0:67203:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b888 lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fef1 expref: -99 pid: 67203 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.605626:0:67203:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b888 lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fef1 expref: -99 pid: 67203 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.605640:0:67203:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b888 lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fef1 expref: -99 pid: 67203 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.605647:0:67203:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b888 lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fef1 expref: -99 pid: 67203 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.605660:0:67203:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b888 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fef1 expref: -99 pid: 67203 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826287.605671:0:67203:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x2b, status=0 req@ffff907d908461c0 x1874670011425152/t8589942978(8589942978) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/1264 e 0 to 0 dl 1787826305 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1787826287.605684:0:67203:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908461c0 x1874670011425152/t8589942978(8589942978) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/1264 e 0 to 0 dl 1787826305 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826287.605698:0:67203:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 656 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b888 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fef1 expref: -99 pid: 67203 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1787826287.605710:0:67203:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b888 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fef1 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.605721:0:67203:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/515 (0 0) 00000080:00200000:0.0:1787826287.605726:0:67203:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x200000404:0x202:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1787826287.605783:0:67203:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x203:0x0](ffff907d93310908): rc = 0 00000080:00200000:0.0:1787826287.605801:0:67203:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x203:0x0]: layout version change: 4294967294 -> 6 00000080:00010000:0.0:1787826287.605806:0:67203:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) for lock 0xe5f736940357b888 00000080:00010000:0.0:1787826287.605817:0:67203:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) 00000080:00200008:0.0:1787826287.605831:0:67203:0:(file.c:1090:ll_file_open()) START file f204c.sanity-flr:[0x200000404:0x203:0x0](ffff907d93310908), flags 100101 00000002:00100000:0.0:1787826287.605852:0:67203:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908461c0 x1874670011425152/t8589942978(8589942978) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1192/1264 e 0 to 0 dl 1787826305 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000080:00010000:0.0:1787826287.605895:0:67203:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b888 from it ffff907d832414e0 00010000:00010000:0.0:1787826287.605900:0:67203:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b888 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0fef1 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.605909:0:67203:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b888 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0fef1 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826287.605928:0:67203:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908), inode mode 81a4 mask 42 00000080:00200008:0.0:1787826287.615897:0:67203:0:(file.c:2652:do_file_write_iter()) START file f204c.sanity-flr:[0x200000404:0x203:0x0], ppos: 0, count: 2097152 00000080:00200000:0.0:1787826287.615912:0:67203:0:(file.c:2679:do_file_write_iter()) switching to DIO 00000080:00200000:0.0:1787826287.615918:0:67203:0:(file.c:2090:ll_file_io_generic()) f204c.sanity-flr: write ppos: 0, bytes: 2097152 00000080:00200000:0.0:1787826287.615931:0:67203:0:(file.c:1848:ll_io_set_mirror()) f204c.sanity-flr: desiginated mirror: 0 00000020:00200000:0.0:1787826287.615939:0:67203:0:(cl_io.c:218:cl_io_rw_init()) header@000000008ae07a1f[0x110, 2, [0x200000404:0x203:0x0] hash] 00000020:00200000:0.0:1787826287.616292:0:67203:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [0, 2097152) 0 0 00000080:00200000:0.0:1787826287.616300:0:67203:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826287.616316:0:67203:0:(lov_io.c:571:lov_io_mirror_init()) [0x200000404:0x203:0x0] need write intent for [0, 2097152) 00000020:00200000:0.0:1787826287.616320:0:67203:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1787826287.616327:0:67203:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 6 need write layout 1, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1787826287.616330:0:67203:0:(vvp_io.c:371:vvp_io_fini()) [0x200000404:0x203:0x0] write layout, type 2 [0x0, 0x200000) 00000080:00010000:0.0:1787826287.616341:0:67203:0:(file.c:7076:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000404:0x203:0x0](ffff907d93310908) 00000002:00010000:0.0:1787826287.616355:0:67203:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x203:0x0]) in obj [0x200000404:0x203:0x0], intent: layout flags 02 00010000:00010000:0.0:1787826287.616390:0:67203:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b888 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0fef1 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826287.616413:0:67203:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x203:0x0]: losing layout lock 00010000:00010000:0.0:1787826287.616459:0:67203:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b888 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0fef1 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826287.616468:0:67203:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826287.616471:0:67203:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b888 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0fef1 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826287.616514:0:67203:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 3/0,1 mode: --/EX res: [0x200000404:0x203:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67203 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.616522:0:67203:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 3/0,1 mode: --/EX res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67203 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826287.616533:0:67203:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826287.616546:0:67203:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908470c0 x1874670011425408/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/2048 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826287.616557:0:67203:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 3/0,1 mode: --/EX res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67203 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826287.616577:0:67203:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:67203:1874670011425408:192.168.203.104@tcp:101:dd.0 00000100:00100000:0.0:1787826287.616691:0:67203:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f91a80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826287.631930:0:67203:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589942979 gen 1 00000100:00100000:0.0:1787826287.631959:0:67203:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d908461c0 x1874670011425152/t8589942978(8589942978) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1192/1264 e 0 to 0 dl 1787826305 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1787826287.631972:0:67203:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:67203:1874670011425408:192.168.203.104@tcp:101:dd.0 00010000:00010000:0.0:1787826287.631999:0:67203:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 4/0,1 mode: --/EX res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.632011:0:67203:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 4/0,1 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.632023:0:67203:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 4/0,1 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.632039:0:67203:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 4/0,1 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.632052:0:67203:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 5/1,1 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.632060:0:67203:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 5/1,1 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.632068:0:67203:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826287.632078:0:67203:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x400 disposition=0x1, status=0 req@ffff907d908470c0 x1874670011425408/t8589942979(8589942979) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/792 e 0 to 0 dl 1787826305 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826287.632095:0:67203:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 448 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1787826287.632106:0:67203:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.632118:0:67203:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/515 (0 0) 00000080:00200000:0.0:1787826287.632124:0:67203:0:(namei.c:952:ll_intent_lock()) intent lock 1024 on i1 [0x200000404:0x203:0x0] suppgids 0 0: rc 0 00000080:00010000:0.0:1787826287.632140:0:67203:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) for lock 0xe5f736940357b88f 00000080:00010000:0.0:1787826287.632149:0:67203:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b88f from it ffff9dd141c3fbb8 00010000:00010000:0.0:1787826287.632152:0:67203:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.632165:0:67203:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826287.632181:0:67203:0:(file.c:2240:ll_file_io_generic()) f204c.sanity-flr: 2 io complete with rc: 0, result: 0, restart: 1 00000080:00200000:0.0:1787826287.632185:0:67203:0:(file.c:2247:ll_file_io_generic()) f204c.sanity-flr: restart write from ppos=0 bytes=2097152 retries=999 ret=0: rc = 0 00000080:00200000:0.0:1787826287.632191:0:67203:0:(file.c:1848:ll_io_set_mirror()) f204c.sanity-flr: desiginated mirror: 0 00000020:00200000:0.0:1787826287.632200:0:67203:0:(cl_io.c:218:cl_io_rw_init()) header@000000008ae07a1f[0x110, 2, [0x200000404:0x203:0x0] hash] 00000020:00200000:0.0:1787826287.632202:0:67203:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [0, 2097152) 0 0 00000080:00200000:0.0:1787826287.632208:0:67203:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 6 restore needed 0 00010000:00010000:0.0:1787826287.632223:0:67203:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826287.632234:0:67203:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:0.0:1787826287.632243:0:67203:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x203:0x0](00000000010d8bed) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826287.632282:0:67203:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826287.632290:0:67203:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826287.632328:0:67203:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826287.632337:0:67203:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 1, move mirror from 1 to 1, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826287.632340:0:67203:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826287.632346:0:67203:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x10 00020000:00200000:0.0:1787826287.632368:0:67203:0:(lov_io.c:1253:lov_io_iter_init()) component[4] flags 0x40 00020000:00200000:0.0:1787826287.632381:0:67203:0:(lov_io.c:1253:lov_io_iter_init()) component[5] flags 0x0 00000080:00200000:0.0:1787826287.632399:0:67203:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1787826287.632412:0:67203:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826287.632420:0:67203:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1787826287.632428:0:67203:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x203:0x0](00000000010d8bed) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826287.632444:0:67203:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826287.632449:0:67203:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 1, move mirror from 1 to 1, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826287.632452:0:67203:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826287.632454:0:67203:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x10 00020000:00200000:0.0:1787826287.632471:0:67203:0:(lov_io.c:1253:lov_io_iter_init()) component[4] flags 0x40 00020000:00200000:0.0:1787826287.632472:0:67203:0:(lov_io.c:1253:lov_io_iter_init()) component[5] flags 0x0 00000080:00200000:0.0:1787826287.632478:0:67203:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1787826287.632545:0:67203:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x203:0x0]: layout version change: 4294967294 -> 7 00010000:00010000:0.0:1787826287.632551:0:67203:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826287.632558:0:67203:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00020000:00200000:0.0:1787826287.632571:0:67203:0:(lov_io.c:311:lov_io_mirror_write_intent()) [0x200000404:0x203:0x0]there are 0 components to be staled to modify file extent [0x0, 0xa00000), iot: 2 00020000:00200000:0.0:1787826287.632574:0:67203:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826287.632579:0:67203:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00000080:00200000:0.0:1787826287.632583:0:67203:0:(file.c:2170:ll_file_io_generic()) Range lock [0, 511] 00020000:00200000:0.0:1787826287.632589:0:67203:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 2097152) 2097152, 2097152 00020000:00200000:0.0:1787826287.632592:0:67203:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1787826287.632603:0:67203:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 511] 00000008:00010000:0.0:1787826287.632681:0:67203:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1787826287.632686:0:67203:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da032e4b0, osc lock ffff907d963694b8, flags 0x0 00000080:00200000:0.0:1787826287.632696:0:67203:0:(vvp_io.c:1339:vvp_io_write_start()) f204c.sanity-flr: write [0, 2097152) append 0 append lockpos 0 00000002:00010000:0.0:1787826287.632720:0:67203:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x0:0x0:0x0]) in obj [0x200000404:0x203:0x0], intent: getxattr flags 0 00010000:00010000:0.0:1787826287.632762:0:67203:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b896 lrc: 3/1,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67203 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.632780:0:67203:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b896 lrc: 3/1,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67203 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826287.632789:0:67203:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b896 lrc: 3/1,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67203 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826287.632809:0:67203:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:67203:1874670011425536:192.168.203.104@tcp:101:dd.0 00000100:00100000:2.0:1787826287.632985:0:67203:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f91a80 going to sleep for 13 seconds 00000100:00100000:1.0:1787826287.638554:0:67203:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:67203:1874670011425536:192.168.203.104@tcp:101:dd.0 00010000:00010000:1.0:1787826287.638581:0:67203:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b896 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff14 expref: -99 pid: 67203 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826287.638597:0:67203:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b896 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ff14 expref: -99 pid: 67203 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826287.638635:0:67203:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b896 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff14 expref: -99 pid: 67203 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826287.638648:0:67203:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x80 disposition=0x0, status=0 req@ffff907d908470c0 x1874670011425536/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 568/1544 e 0 to 0 dl 1787826305 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826287.638659:0:67203:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b896 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff14 expref: -99 pid: 67203 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826287.638671:0:67203:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/515 (0 0) 00000080:00200000:1.0:1787826287.638691:0:67203:0:(namei.c:952:ll_intent_lock()) intent lock 128 on i1 [0x200000404:0x203:0x0] suppgids 0 -1: rc 0 00000080:00010000:1.0:1787826287.638715:0:67203:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) for lock 0xe5f736940357b896 00000080:00010000:1.0:1787826287.638722:0:67203:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b896 from it ffff9dd141c3f8e8 00010000:00010000:1.0:1787826287.638726:0:67203:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b896 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ff14 expref: -99 pid: 67203 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826287.638736:0:67203:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b896 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff14 expref: -99 pid: 67203 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826287.638783:0:67203:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908), size=2097152 (max 335544320), offset=0=0x0, pages 512 (max 81920), lockless, parallel, hybrid 00000020:00200000:1.0:1787826287.639780:0:67203:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 512 items remaining. 00000080:00200000:1.0:1787826287.639786:0:67203:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 2097152 count 2097152 file_offset 2097152 00000020:00010000:1.0:1787826287.639797:0:67203:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b1d58 (ffff907d884a7b48/0) at cl_lock_release():245 00000020:00010000:1.0:1787826287.639800:0:67203:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b1d58 (ffff907d884a7b48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826287.639805:0:67203:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da032e4b0 (ffff907d884a7098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826287.639807:0:67203:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b1d58 (ffff907d884a7b48/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826287.639811:0:67203:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da032e4b0 (ffff907d884a7b48/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826287.639819:0:67203:0:(vvp_io.c:533:vvp_io_advance()) advancing 2097152 bytes 00000020:00200000:1.0:1787826287.639825:0:67203:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1787826287.639847:0:58785:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907d908aa900 00000008:00100000:1.0:1787826287.642094:0:58785:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d908a1a40 - object 0x240000400:806 offset 0<>2097152 00000100:00100000:0.0:1787826287.644529:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011425664:192.168.203.104@tcp:4:dd.0 00000100:00100000:0.0:1787826287.710438:0:2349:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff907d986d0000: committing for last_committed 21474840502 gen 3 00000008:00010000:0.0:1787826287.710520:0:2349:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907d908aa900 00000020:00200000:0.0:1787826287.710526:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 511 items remaining. 00000020:00200000:0.0:1787826287.710529:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 510 items remaining. 00000020:00200000:0.0:1787826287.710531:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 509 items remaining. 00000020:00200000:0.0:1787826287.710532:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 508 items remaining. 00000020:00200000:0.0:1787826287.710534:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 507 items remaining. 00000020:00200000:0.0:1787826287.710535:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 506 items remaining. 00000020:00200000:0.0:1787826287.710537:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 505 items remaining. 00000020:00200000:0.0:1787826287.710538:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 504 items remaining. 00000020:00200000:0.0:1787826287.710539:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 503 items remaining. 00000020:00200000:0.0:1787826287.710542:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 502 items remaining. 00000020:00200000:0.0:1787826287.710543:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 501 items remaining. 00000020:00200000:0.0:1787826287.710544:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 500 items remaining. 00000020:00200000:0.0:1787826287.710546:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 499 items remaining. 00000020:00200000:0.0:1787826287.710548:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 498 items remaining. 00000020:00200000:0.0:1787826287.710549:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 497 items remaining. 00000020:00200000:0.0:1787826287.710551:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 496 items remaining. 00000020:00200000:0.0:1787826287.710552:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 495 items remaining. 00000020:00200000:0.0:1787826287.710553:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 494 items remaining. 00000020:00200000:0.0:1787826287.710555:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 493 items remaining. 00000020:00200000:0.0:1787826287.710556:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 492 items remaining. 00000020:00200000:0.0:1787826287.710558:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 491 items remaining. 00000020:00200000:0.0:1787826287.710559:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 490 items remaining. 00000020:00200000:0.0:1787826287.710561:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 489 items remaining. 00000020:00200000:0.0:1787826287.710562:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 488 items remaining. 00000020:00200000:0.0:1787826287.710564:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 487 items remaining. 00000020:00200000:0.0:1787826287.710565:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 486 items remaining. 00000020:00200000:0.0:1787826287.710566:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 485 items remaining. 00000020:00200000:0.0:1787826287.710568:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 484 items remaining. 00000020:00200000:0.0:1787826287.710570:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 483 items remaining. 00000020:00200000:0.0:1787826287.710571:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 482 items remaining. 00000020:00200000:0.0:1787826287.710573:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 481 items remaining. 00000020:00200000:0.0:1787826287.710574:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 480 items remaining. 00000020:00200000:0.0:1787826287.710575:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 479 items remaining. 00000020:00200000:0.0:1787826287.710576:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 478 items remaining. 00000020:00200000:0.0:1787826287.710578:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 477 items remaining. 00000020:00200000:0.0:1787826287.710579:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 476 items remaining. 00000020:00200000:0.0:1787826287.710580:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 475 items remaining. 00000020:00200000:0.0:1787826287.710582:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 474 items remaining. 00000020:00200000:0.0:1787826287.710583:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 473 items remaining. 00000020:00200000:0.0:1787826287.710584:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 472 items remaining. 00000020:00200000:0.0:1787826287.710586:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 471 items remaining. 00000020:00200000:0.0:1787826287.710588:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 470 items remaining. 00000020:00200000:0.0:1787826287.710589:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 469 items remaining. 00000020:00200000:0.0:1787826287.710591:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 468 items remaining. 00000020:00200000:0.0:1787826287.710593:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 467 items remaining. 00000020:00200000:0.0:1787826287.710594:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 466 items remaining. 00000020:00200000:0.0:1787826287.710596:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 465 items remaining. 00000020:00200000:0.0:1787826287.710597:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 464 items remaining. 00000020:00200000:0.0:1787826287.710599:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 463 items remaining. 00000020:00200000:0.0:1787826287.710600:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 462 items remaining. 00000020:00200000:0.0:1787826287.710602:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 461 items remaining. 00000020:00200000:0.0:1787826287.710603:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 460 items remaining. 00000020:00200000:0.0:1787826287.710605:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 459 items remaining. 00000020:00200000:0.0:1787826287.710653:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 458 items remaining. 00000020:00200000:0.0:1787826287.710655:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 457 items remaining. 00000020:00200000:0.0:1787826287.710656:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 456 items remaining. 00000020:00200000:0.0:1787826287.710658:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 455 items remaining. 00000020:00200000:0.0:1787826287.710659:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 454 items remaining. 00000020:00200000:0.0:1787826287.710660:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 453 items remaining. 00000020:00200000:0.0:1787826287.710662:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 452 items remaining. 00000020:00200000:0.0:1787826287.710663:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 451 items remaining. 00000020:00200000:0.0:1787826287.710665:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 450 items remaining. 00000020:00200000:0.0:1787826287.710666:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 449 items remaining. 00000020:00200000:0.0:1787826287.710667:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 448 items remaining. 00000020:00200000:0.0:1787826287.710668:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 447 items remaining. 00000020:00200000:0.0:1787826287.710669:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 446 items remaining. 00000020:00200000:0.0:1787826287.710671:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 445 items remaining. 00000020:00200000:0.0:1787826287.710672:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 444 items remaining. 00000020:00200000:0.0:1787826287.710673:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 443 items remaining. 00000020:00200000:0.0:1787826287.710675:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 442 items remaining. 00000020:00200000:0.0:1787826287.710676:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 441 items remaining. 00000020:00200000:0.0:1787826287.710677:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 440 items remaining. 00000020:00200000:0.0:1787826287.710678:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 439 items remaining. 00000020:00200000:0.0:1787826287.710680:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 438 items remaining. 00000020:00200000:0.0:1787826287.710682:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 437 items remaining. 00000020:00200000:0.0:1787826287.710683:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 436 items remaining. 00000020:00200000:0.0:1787826287.710685:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 435 items remaining. 00000020:00200000:0.0:1787826287.710686:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 434 items remaining. 00000020:00200000:0.0:1787826287.710687:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 433 items remaining. 00000020:00200000:0.0:1787826287.710689:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 432 items remaining. 00000020:00200000:0.0:1787826287.710690:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 431 items remaining. 00000020:00200000:0.0:1787826287.710691:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 430 items remaining. 00000020:00200000:0.0:1787826287.710693:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 429 items remaining. 00000020:00200000:0.0:1787826287.710695:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 428 items remaining. 00000020:00200000:0.0:1787826287.710696:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 427 items remaining. 00000020:00200000:0.0:1787826287.710697:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 426 items remaining. 00000020:00200000:0.0:1787826287.710699:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 425 items remaining. 00000020:00200000:0.0:1787826287.710700:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 424 items remaining. 00000020:00200000:0.0:1787826287.710702:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 423 items remaining. 00000020:00200000:0.0:1787826287.710703:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 422 items remaining. 00000020:00200000:0.0:1787826287.710705:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 421 items remaining. 00000020:00200000:0.0:1787826287.710706:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 420 items remaining. 00000020:00200000:0.0:1787826287.710708:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 419 items remaining. 00000020:00200000:0.0:1787826287.710709:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 418 items remaining. 00000020:00200000:0.0:1787826287.710711:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 417 items remaining. 00000020:00200000:0.0:1787826287.710712:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 416 items remaining. 00000020:00200000:0.0:1787826287.710713:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 415 items remaining. 00000020:00200000:0.0:1787826287.710715:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 414 items remaining. 00000020:00200000:0.0:1787826287.710716:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 413 items remaining. 00000020:00200000:0.0:1787826287.710717:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 412 items remaining. 00000020:00200000:0.0:1787826287.710719:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 411 items remaining. 00000020:00200000:0.0:1787826287.710720:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 410 items remaining. 00000020:00200000:0.0:1787826287.710721:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 409 items remaining. 00000020:00200000:0.0:1787826287.710723:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 408 items remaining. 00000020:00200000:0.0:1787826287.710724:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 407 items remaining. 00000020:00200000:0.0:1787826287.710726:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 406 items remaining. 00000020:00200000:0.0:1787826287.710727:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 405 items remaining. 00000020:00200000:0.0:1787826287.710729:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 404 items remaining. 00000020:00200000:0.0:1787826287.710730:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 403 items remaining. 00000020:00200000:0.0:1787826287.710731:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 402 items remaining. 00000020:00200000:0.0:1787826287.710732:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 401 items remaining. 00000020:00200000:0.0:1787826287.710734:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 400 items remaining. 00000020:00200000:0.0:1787826287.710735:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 399 items remaining. 00000020:00200000:0.0:1787826287.710737:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 398 items remaining. 00000020:00200000:0.0:1787826287.710738:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 397 items remaining. 00000020:00200000:0.0:1787826287.710739:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 396 items remaining. 00000020:00200000:0.0:1787826287.710741:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 395 items remaining. 00000020:00200000:0.0:1787826287.710743:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 394 items remaining. 00000020:00200000:0.0:1787826287.710745:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 393 items remaining. 00000020:00200000:0.0:1787826287.710746:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 392 items remaining. 00000020:00200000:0.0:1787826287.710747:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 391 items remaining. 00000020:00200000:0.0:1787826287.710749:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 390 items remaining. 00000020:00200000:0.0:1787826287.710751:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 389 items remaining. 00000020:00200000:0.0:1787826287.710753:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 388 items remaining. 00000020:00200000:0.0:1787826287.710754:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 387 items remaining. 00000020:00200000:0.0:1787826287.710756:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 386 items remaining. 00000020:00200000:0.0:1787826287.710758:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 385 items remaining. 00000020:00200000:0.0:1787826287.710759:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 384 items remaining. 00000020:00200000:0.0:1787826287.710761:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 383 items remaining. 00000020:00200000:0.0:1787826287.710762:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 382 items remaining. 00000020:00200000:0.0:1787826287.710764:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 381 items remaining. 00000020:00200000:0.0:1787826287.710765:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 380 items remaining. 00000020:00200000:0.0:1787826287.710767:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 379 items remaining. 00000020:00200000:0.0:1787826287.710768:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 378 items remaining. 00000020:00200000:0.0:1787826287.710769:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 377 items remaining. 00000020:00200000:0.0:1787826287.710771:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 376 items remaining. 00000020:00200000:0.0:1787826287.710772:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 375 items remaining. 00000020:00200000:0.0:1787826287.710774:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 374 items remaining. 00000020:00200000:0.0:1787826287.710776:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 373 items remaining. 00000020:00200000:0.0:1787826287.710777:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 372 items remaining. 00000020:00200000:0.0:1787826287.710779:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 371 items remaining. 00000020:00200000:0.0:1787826287.710780:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 370 items remaining. 00000020:00200000:0.0:1787826287.710782:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 369 items remaining. 00000020:00200000:0.0:1787826287.710783:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 368 items remaining. 00000020:00200000:0.0:1787826287.710785:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 367 items remaining. 00000020:00200000:0.0:1787826287.710786:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 366 items remaining. 00000020:00200000:0.0:1787826287.710788:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 365 items remaining. 00000020:00200000:0.0:1787826287.710789:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 364 items remaining. 00000020:00200000:0.0:1787826287.710791:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 363 items remaining. 00000020:00200000:0.0:1787826287.710792:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 362 items remaining. 00000020:00200000:0.0:1787826287.710794:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 361 items remaining. 00000020:00200000:0.0:1787826287.710795:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 360 items remaining. 00000020:00200000:0.0:1787826287.710797:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 359 items remaining. 00000020:00200000:0.0:1787826287.710798:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 358 items remaining. 00000020:00200000:0.0:1787826287.710800:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 357 items remaining. 00000020:00200000:0.0:1787826287.710801:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 356 items remaining. 00000020:00200000:0.0:1787826287.710803:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 355 items remaining. 00000020:00200000:0.0:1787826287.710804:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 354 items remaining. 00000020:00200000:0.0:1787826287.710806:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 353 items remaining. 00000020:00200000:0.0:1787826287.710808:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 352 items remaining. 00000020:00200000:0.0:1787826287.710809:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 351 items remaining. 00000020:00200000:0.0:1787826287.710811:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 350 items remaining. 00000020:00200000:0.0:1787826287.710813:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 349 items remaining. 00000020:00200000:0.0:1787826287.710814:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 348 items remaining. 00000020:00200000:0.0:1787826287.710816:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 347 items remaining. 00000020:00200000:0.0:1787826287.710817:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 346 items remaining. 00000020:00200000:0.0:1787826287.710819:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 345 items remaining. 00000020:00200000:0.0:1787826287.710820:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 344 items remaining. 00000020:00200000:0.0:1787826287.710822:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 343 items remaining. 00000020:00200000:0.0:1787826287.710824:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 342 items remaining. 00000020:00200000:0.0:1787826287.710826:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 341 items remaining. 00000020:00200000:0.0:1787826287.710828:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 340 items remaining. 00000020:00200000:0.0:1787826287.710830:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 339 items remaining. 00000020:00200000:0.0:1787826287.710831:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 338 items remaining. 00000020:00200000:0.0:1787826287.710832:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 337 items remaining. 00000020:00200000:0.0:1787826287.710834:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 336 items remaining. 00000020:00200000:0.0:1787826287.710836:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 335 items remaining. 00000020:00200000:0.0:1787826287.710837:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 334 items remaining. 00000020:00200000:0.0:1787826287.710838:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 333 items remaining. 00000020:00200000:0.0:1787826287.710840:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 332 items remaining. 00000020:00200000:0.0:1787826287.710841:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 331 items remaining. 00000020:00200000:0.0:1787826287.710843:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 330 items remaining. 00000020:00200000:0.0:1787826287.710844:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 329 items remaining. 00000020:00200000:0.0:1787826287.710846:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 328 items remaining. 00000020:00200000:0.0:1787826287.710848:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 327 items remaining. 00000020:00200000:0.0:1787826287.710849:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 326 items remaining. 00000020:00200000:0.0:1787826287.710851:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 325 items remaining. 00000020:00200000:0.0:1787826287.710853:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 324 items remaining. 00000020:00200000:0.0:1787826287.710854:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 323 items remaining. 00000020:00200000:0.0:1787826287.710856:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 322 items remaining. 00000020:00200000:0.0:1787826287.710857:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 321 items remaining. 00000020:00200000:0.0:1787826287.710859:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 320 items remaining. 00000020:00200000:0.0:1787826287.710860:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 319 items remaining. 00000020:00200000:0.0:1787826287.710862:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 318 items remaining. 00000020:00200000:0.0:1787826287.710863:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 317 items remaining. 00000020:00200000:0.0:1787826287.710865:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 316 items remaining. 00000020:00200000:0.0:1787826287.710866:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 315 items remaining. 00000020:00200000:0.0:1787826287.710868:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 314 items remaining. 00000020:00200000:0.0:1787826287.710869:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 313 items remaining. 00000020:00200000:0.0:1787826287.710871:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 312 items remaining. 00000020:00200000:0.0:1787826287.710872:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 311 items remaining. 00000020:00200000:0.0:1787826287.710874:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 310 items remaining. 00000020:00200000:0.0:1787826287.710876:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 309 items remaining. 00000020:00200000:0.0:1787826287.710877:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 308 items remaining. 00000020:00200000:0.0:1787826287.710879:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 307 items remaining. 00000020:00200000:0.0:1787826287.710880:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 306 items remaining. 00000020:00200000:0.0:1787826287.710882:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 305 items remaining. 00000020:00200000:0.0:1787826287.710883:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 304 items remaining. 00000020:00200000:0.0:1787826287.710885:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 303 items remaining. 00000020:00200000:0.0:1787826287.710886:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 302 items remaining. 00000020:00200000:0.0:1787826287.710888:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 301 items remaining. 00000020:00200000:0.0:1787826287.710890:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 300 items remaining. 00000020:00200000:0.0:1787826287.710892:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 299 items remaining. 00000020:00200000:0.0:1787826287.710893:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 298 items remaining. 00000020:00200000:0.0:1787826287.710895:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 297 items remaining. 00000020:00200000:0.0:1787826287.710897:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 296 items remaining. 00000020:00200000:0.0:1787826287.710899:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 295 items remaining. 00000020:00200000:0.0:1787826287.710901:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 294 items remaining. 00000020:00200000:0.0:1787826287.710902:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 293 items remaining. 00000020:00200000:0.0:1787826287.710904:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 292 items remaining. 00000020:00200000:0.0:1787826287.710906:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 291 items remaining. 00000020:00200000:0.0:1787826287.710907:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 290 items remaining. 00000020:00200000:0.0:1787826287.710909:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 289 items remaining. 00000020:00200000:0.0:1787826287.710910:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 288 items remaining. 00000020:00200000:0.0:1787826287.710912:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 287 items remaining. 00000020:00200000:0.0:1787826287.710914:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 286 items remaining. 00000020:00200000:0.0:1787826287.710915:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 285 items remaining. 00000020:00200000:0.0:1787826287.710917:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 284 items remaining. 00000020:00200000:0.0:1787826287.710918:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 283 items remaining. 00000020:00200000:0.0:1787826287.710920:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 282 items remaining. 00000020:00200000:0.0:1787826287.710922:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 281 items remaining. 00000020:00200000:0.0:1787826287.710923:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 280 items remaining. 00000020:00200000:0.0:1787826287.710924:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 279 items remaining. 00000020:00200000:0.0:1787826287.710927:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 278 items remaining. 00000020:00200000:0.0:1787826287.710928:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 277 items remaining. 00000020:00200000:0.0:1787826287.710930:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 276 items remaining. 00000020:00200000:0.0:1787826287.710932:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 275 items remaining. 00000020:00200000:0.0:1787826287.710933:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 274 items remaining. 00000020:00200000:0.0:1787826287.710935:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 273 items remaining. 00000020:00200000:0.0:1787826287.710937:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 272 items remaining. 00000020:00200000:0.0:1787826287.710938:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 271 items remaining. 00000020:00200000:0.0:1787826287.710940:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 270 items remaining. 00000020:00200000:0.0:1787826287.710942:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 269 items remaining. 00000020:00200000:0.0:1787826287.710943:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 268 items remaining. 00000020:00200000:0.0:1787826287.710945:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 267 items remaining. 00000020:00200000:0.0:1787826287.710947:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 266 items remaining. 00000020:00200000:0.0:1787826287.710948:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 265 items remaining. 00000020:00200000:0.0:1787826287.710950:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 264 items remaining. 00000020:00200000:0.0:1787826287.710951:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 263 items remaining. 00000020:00200000:0.0:1787826287.710953:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 262 items remaining. 00000020:00200000:0.0:1787826287.710954:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 261 items remaining. 00000020:00200000:0.0:1787826287.710956:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 260 items remaining. 00000020:00200000:0.0:1787826287.710957:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 259 items remaining. 00000020:00200000:0.0:1787826287.710959:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 258 items remaining. 00000020:00200000:0.0:1787826287.710961:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 257 items remaining. 00000020:00200000:0.0:1787826287.710962:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000020:00200000:0.0:1787826287.710963:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:0.0:1787826287.710965:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:0.0:1787826287.710966:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:0.0:1787826287.710967:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:0.0:1787826287.710969:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:0.0:1787826287.710970:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:0.0:1787826287.710972:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:0.0:1787826287.710973:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:0.0:1787826287.710974:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:0.0:1787826287.710988:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:0.0:1787826287.710991:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:0.0:1787826287.710992:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:0.0:1787826287.710994:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:0.0:1787826287.710995:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:0.0:1787826287.710996:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:0.0:1787826287.710998:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:0.0:1787826287.711000:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:0.0:1787826287.711001:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:0.0:1787826287.711002:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:0.0:1787826287.711004:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:0.0:1787826287.711005:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:0.0:1787826287.711007:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:0.0:1787826287.711009:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:0.0:1787826287.711010:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:0.0:1787826287.711012:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:0.0:1787826287.711014:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:0.0:1787826287.711015:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:0.0:1787826287.711016:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:0.0:1787826287.711017:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:0.0:1787826287.711019:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:0.0:1787826287.711020:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:0.0:1787826287.711022:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:0.0:1787826287.711023:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:0.0:1787826287.711025:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:0.0:1787826287.711026:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:0.0:1787826287.711028:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:0.0:1787826287.711029:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:0.0:1787826287.711030:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:0.0:1787826287.711032:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:0.0:1787826287.711033:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:0.0:1787826287.711034:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:0.0:1787826287.711036:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:0.0:1787826287.711038:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:0.0:1787826287.711039:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:0.0:1787826287.711041:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:0.0:1787826287.711042:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:0.0:1787826287.711043:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:0.0:1787826287.711045:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:0.0:1787826287.711046:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:0.0:1787826287.711047:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:0.0:1787826287.711049:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:0.0:1787826287.711050:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:0.0:1787826287.711051:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:0.0:1787826287.711053:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:0.0:1787826287.711055:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:0.0:1787826287.711056:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:0.0:1787826287.711058:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:0.0:1787826287.711060:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:0.0:1787826287.711062:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:0.0:1787826287.711063:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:0.0:1787826287.711064:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:0.0:1787826287.711066:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:0.0:1787826287.711067:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:0.0:1787826287.711069:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:0.0:1787826287.711070:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:0.0:1787826287.711072:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:0.0:1787826287.711074:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:0.0:1787826287.711075:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:0.0:1787826287.711077:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:0.0:1787826287.711079:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:0.0:1787826287.711080:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:0.0:1787826287.711082:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:0.0:1787826287.711084:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:0.0:1787826287.711086:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:0.0:1787826287.711088:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:0.0:1787826287.711089:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:0.0:1787826287.711091:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:0.0:1787826287.711092:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:0.0:1787826287.711093:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:0.0:1787826287.711095:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:0.0:1787826287.711096:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:0.0:1787826287.711098:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:0.0:1787826287.711099:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:0.0:1787826287.711101:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:0.0:1787826287.711102:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:0.0:1787826287.711104:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:0.0:1787826287.711105:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:0.0:1787826287.711107:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:0.0:1787826287.711108:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:0.0:1787826287.711109:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:0.0:1787826287.711111:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:0.0:1787826287.711113:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:0.0:1787826287.711114:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:0.0:1787826287.711116:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:0.0:1787826287.711117:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:0.0:1787826287.711119:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:0.0:1787826287.711120:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:0.0:1787826287.711122:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:0.0:1787826287.711123:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:0.0:1787826287.711124:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:0.0:1787826287.711126:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:0.0:1787826287.711127:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:0.0:1787826287.711129:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:0.0:1787826287.711130:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:0.0:1787826287.711132:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:0.0:1787826287.711134:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:0.0:1787826287.711136:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:0.0:1787826287.711137:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:0.0:1787826287.711139:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:0.0:1787826287.711140:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:0.0:1787826287.711142:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:0.0:1787826287.711143:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:0.0:1787826287.711145:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:0.0:1787826287.711146:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:0.0:1787826287.711148:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:0.0:1787826287.711149:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:0.0:1787826287.711151:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:0.0:1787826287.711152:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:0.0:1787826287.711154:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:0.0:1787826287.711156:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:0.0:1787826287.711158:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:0.0:1787826287.711160:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:0.0:1787826287.711161:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:0.0:1787826287.711162:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:0.0:1787826287.711164:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:0.0:1787826287.711166:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:0.0:1787826287.711167:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:0.0:1787826287.711169:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:0.0:1787826287.711171:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:0.0:1787826287.711172:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:0.0:1787826287.711174:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:0.0:1787826287.711175:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:0.0:1787826287.711177:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:0.0:1787826287.711179:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:0.0:1787826287.711180:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:0.0:1787826287.711183:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:0.0:1787826287.711184:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:0.0:1787826287.711186:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:0.0:1787826287.711188:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:0.0:1787826287.711189:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:0.0:1787826287.711191:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:0.0:1787826287.711193:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:0.0:1787826287.711195:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:0.0:1787826287.711197:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:0.0:1787826287.711198:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:0.0:1787826287.711200:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:0.0:1787826287.711325:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:0.0:1787826287.711329:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:0.0:1787826287.711331:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:0.0:1787826287.711333:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:0.0:1787826287.711335:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:0.0:1787826287.711336:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:0.0:1787826287.711338:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:0.0:1787826287.711340:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:0.0:1787826287.711341:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:0.0:1787826287.711343:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:0.0:1787826287.711345:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:0.0:1787826287.711347:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:0.0:1787826287.711348:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:0.0:1787826287.711350:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:0.0:1787826287.711352:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:0.0:1787826287.711353:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:0.0:1787826287.711355:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:0.0:1787826287.711357:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:0.0:1787826287.711358:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:0.0:1787826287.711360:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:0.0:1787826287.711362:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:0.0:1787826287.711364:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:0.0:1787826287.711365:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:0.0:1787826287.711367:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:0.0:1787826287.711381:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:0.0:1787826287.711382:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:0.0:1787826287.711384:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:0.0:1787826287.711385:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:0.0:1787826287.711388:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:0.0:1787826287.711389:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:0.0:1787826287.711391:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:0.0:1787826287.711393:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:0.0:1787826287.711395:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:0.0:1787826287.711398:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:0.0:1787826287.711399:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:0.0:1787826287.711401:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:0.0:1787826287.711402:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:0.0:1787826287.711404:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:0.0:1787826287.711405:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:0.0:1787826287.711407:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:0.0:1787826287.711409:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:0.0:1787826287.711411:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:0.0:1787826287.711412:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:0.0:1787826287.711414:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:0.0:1787826287.711415:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:0.0:1787826287.711417:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:0.0:1787826287.711419:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:0.0:1787826287.711420:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:0.0:1787826287.711422:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:0.0:1787826287.711424:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:0.0:1787826287.711426:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:0.0:1787826287.711428:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:0.0:1787826287.711430:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:0.0:1787826287.711432:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:0.0:1787826287.711434:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:0.0:1787826287.711436:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:0.0:1787826287.711438:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:0.0:1787826287.711440:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:0.0:1787826287.711441:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:0.0:1787826287.711443:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:0.0:1787826287.711444:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:0.0:1787826287.711446:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:0.0:1787826287.711447:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:0.0:1787826287.711449:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:0.0:1787826287.711450:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:0.0:1787826287.711452:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:0.0:1787826287.711453:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:0.0:1787826287.711455:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:0.0:1787826287.711456:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:0.0:1787826287.711458:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:0.0:1787826287.711460:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:0.0:1787826287.711461:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:0.0:1787826287.711463:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:0.0:1787826287.711464:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:0.0:1787826287.711466:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:0.0:1787826287.711467:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:0.0:1787826287.711469:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:0.0:1787826287.711470:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:0.0:1787826287.711472:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:0.0:1787826287.711474:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:0.0:1787826287.711475:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:0.0:1787826287.711477:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:0.0:1787826287.711478:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:0.0:1787826287.711480:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:0.0:1787826287.711482:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:0.0:1787826287.711483:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:0.0:1787826287.711485:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:0.0:1787826287.711486:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:0.0:1787826287.711488:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:0.0:1787826287.711489:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:0.0:1787826287.711491:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:0.0:1787826287.711493:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:0.0:1787826287.711494:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:0.0:1787826287.711495:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:0.0:1787826287.711497:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:0.0:1787826287.711498:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:0.0:1787826287.711500:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:0.0:1787826287.711501:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:0.0:1787826287.711503:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:0.0:1787826287.711504:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:0.0:1787826287.711506:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:0.0:1787826287.711507:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:0.0:1787826287.711509:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:0.0:1787826287.711511:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:0.0:1787826287.711513:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:0.0:1787826287.711514:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:0.0:1787826287.711516:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:0.0:1787826287.711517:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:0.0:1787826287.711519:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:0.0:1787826287.711520:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1787826287.711833:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1787826287.711848:0:2349:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907d908aa900 00000100:00100000:0.0:1787826287.711877:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011425664:192.168.203.104@tcp:4:dd.0 00000080:00200000:1.0:1787826287.712874:0:67203:0:(file.c:2207:ll_file_io_generic()) Range unlock [0, 511] 00000080:00200000:1.0:1787826287.712889:0:67203:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826287.712895:0:67203:0:(file.c:2240:ll_file_io_generic()) f204c.sanity-flr: 2 io complete with rc: 0, result: 2097152, restart: 0 00000080:00200000:1.0:1787826287.712909:0:67203:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000020:00200000:1.0:1787826287.712913:0:67203:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826287.712923:0:67203:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 2097152 00000080:00200008:1.0:1787826287.712969:0:67203:0:(file.c:479:ll_file_release()) START file f204c.sanity-flr:[0x200000404:0x203:0x0](ffff907d93310908), flags 100001 00000002:00100000:1.0:1787826287.713053:0:67203:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d908461c0 x1874670011425152/t8589942978(8589942978) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1192/1264 e 0 to 0 dl 1787826305 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826287.713066:0:67203:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826287.713071:0:67203:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0b40 x1874670011425920/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826287.713095:0:67203:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:67203:1874670011425920:192.168.203.104@tcp:35:dd.0 00000100:00100000:1.0:1787826287.713345:0:67203:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d96147000 going to sleep for 11 seconds 00000100:00100000:1.0:1787826287.716593:0:67203:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:67203:1874670011425920:192.168.203.104@tcp:35:dd.0 00000002:00100000:1.0:1787826287.716643:0:67203:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d908461c0 x1874670011425152/t8589942978(8589942978) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1192/1264 e 0 to 0 dl 1787826305 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1787826288.510076:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826289.528399:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826290.548900:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826291.579119:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826292.468933:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011426176:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826292.469069:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011426432:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826292.469126:0:2351:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1787826292.469133:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011426048:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826292.469162:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011426304:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826292.473287:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011426304:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826292.473322:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011426176:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826292.473346:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011426432:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826292.473378:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011426048:192.168.203.104@tcp:400:kworker.0 00010000:00010000:2.0:1787826292.596968:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826293.620842:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826294.647864:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826295.669565:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826296.693358:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826297.588936:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011426688:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826297.589051:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011426944:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826297.589103:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1787826297.589110:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011426560:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826297.589137:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011426816:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826297.594343:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011426560:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826297.594366:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011426816:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826297.594379:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011426944:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826297.595511:0:2349:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589942981 gen 1 00000100:00100000:0.0:1787826297.595523:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011426688:192.168.203.104@tcp:400:kworker.0 00010000:00010000:0.0:1787826297.717250:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826298.740770:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826299.765726:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1787826300.007583:0:67225:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826300.007934:0:67225:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826300.007951:0:67225:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1787826300.007965:0:67225:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1787826300.007970:0:67225:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826300.007978:0:67225:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826300.007981:0:67225:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:2.0:1787826300.007987:0:67225:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141d13bf8 00010000:00010000:2.0:1787826300.007989:0:67225:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1787826300.007995:0:67225:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1787826300.008007:0:67225:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826300.008014:0:67225:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=27 00000080:00200000:2.0:1787826300.008018:0:67225:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 1 00000080:00200000:2.0:1787826300.008029:0:67225:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), file ffff907d83b1ef00, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1787826300.008033:0:67225:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), intent=open 00000002:00010000:2.0:1787826300.008045:0:67225:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x202:0x0], intent: open flags 0304001 00010000:00010000:2.0:1787826300.008098:0:67225:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b89d lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.008106:0:67225:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b89d lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826300.008117:0:67225:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826300.008130:0:67225:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114a1c0 x1874670011427200/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826300.008139:0:67225:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b89d lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826300.008161:0:67225:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67225:1874670011427200:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826300.008366:0:67225:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d888670c0 going to sleep for 13 seconds 00000100:00100000:2.0:1787826300.017706:0:67225:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67225:1874670011427200:192.168.203.104@tcp:101:lfs.0 00010000:00010000:2.0:1787826300.017730:0:67225:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b89d lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.017745:0:67225:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b89d lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.017753:0:67225:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b89d lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.017766:0:67225:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b89d lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.017775:0:67225:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b89d lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.017781:0:67225:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8629c200 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:2.0:1787826300.017786:0:67225:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b89d lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.017804:0:67225:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b89d lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.017813:0:67225:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b89d lrc: 5/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.017820:0:67225:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1787826300.017826:0:67225:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b89d lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.017833:0:67225:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b89d lrc: 0/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826300.017851:0:67225:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d9114a1c0 x1874670011427200/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826318 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1787826300.017865:0:67225:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x202:0x0] suppgids 0 -1: rc -20 00000080:00200000:3.0:1787826300.019233:0:67225:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826300.019267:0:67225:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826300.019297:0:67225:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1787826300.019313:0:67225:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:3.0:1787826300.019322:0:67225:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826300.019328:0:67225:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826300.019332:0:67225:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:3.0:1787826300.019338:0:67225:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141d13bf8 00010000:00010000:3.0:1787826300.019342:0:67225:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:3.0:1787826300.019351:0:67225:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:3.0:1787826300.019365:0:67225:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826300.019373:0:67225:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=267 00000080:00200000:3.0:1787826300.019379:0:67225:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 44 00000080:00200008:3.0:1787826300.019387:0:67225:0:(file.c:1090:ll_file_open()) START file d204c.sanity-flr:[0x200000404:0x202:0x0](ffff907d93313248), flags 304000 00000002:00010000:3.0:1787826300.019401:0:67225:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x202:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:3.0:1787826300.019446:0:67225:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef000/0xe5f736940357b8a4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826300.019457:0:67225:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef000/0xe5f736940357b8a4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826300.019468:0:67225:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826300.019478:0:67225:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d835252c0 x1874670011427456/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826300.019486:0:67225:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef000/0xe5f736940357b8a4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826300.019507:0:67225:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67225:1874670011427456:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826300.019795:0:67225:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb840 going to sleep for 13 seconds 00000100:00100000:1.0:1787826300.026812:0:67225:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67225:1874670011427456:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826300.026839:0:67225:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef000/0xe5f736940357b8a4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.026853:0:67225:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef000/0xe5f736940357b8a4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.026860:0:67225:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef000/0xe5f736940357b8a4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.026868:0:67225:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef000/0xe5f736940357b8a4 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.026878:0:67225:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef000/0xe5f736940357b8a4 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.026885:0:67225:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907da59ef000 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826300.026889:0:67225:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef000/0xe5f736940357b8a4 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.026897:0:67225:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef000/0xe5f736940357b8a4 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.026905:0:67225:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef000/0xe5f736940357b8a4 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.026914:0:67225:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826300.026921:0:67225:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef000/0xe5f736940357b8a4 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.026928:0:67225:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef000/0xe5f736940357b8a4 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 67225 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826300.026947:0:67225:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffff907d835252c0 x1874670011427456/t8589942982(8589942982) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826318 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826300.026962:0:67225:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d835252c0 x1874670011427456/t8589942982(8589942982) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826318 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826300.026973:0:67225:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826300.027002:0:67225:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x202:0x0], updating i_size 4096 00000002:00100000:1.0:1787826300.027021:0:67225:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d835252c0 x1874670011427456/t8589942982(8589942982) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826318 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1787826300.027136:0:67225:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248) cmd=c0086915 arg=23123d0 00000100:00100000:1.0:1787826300.027215:0:67225:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67225:1874670011427584:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826300.027324:0:67225:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d96147000 going to sleep for 13 seconds 00000100:00100000:1.0:1787826300.031457:0:67225:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67225:1874670011427584:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826300.031794:0:67225:0:(file.c:479:ll_file_release()) START file d204c.sanity-flr:[0x200000404:0x202:0x0](ffff907d93313248), flags 304000 00000002:00100000:1.0:1787826300.031869:0:67225:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d835252c0 x1874670011427456/t8589942982(8589942982) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826318 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826300.031882:0:67225:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826300.031888:0:67225:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0b40 x1874670011427840/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826300.031912:0:67225:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67225:1874670011427840:192.168.203.104@tcp:35:lfs.0 00000100:00100000:1.0:1787826300.031992:0:67225:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d96147000 going to sleep for 11 seconds 00000100:00100000:1.0:1787826300.036708:0:67225:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67225:1874670011427840:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826300.036742:0:67225:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d835252c0 x1874670011427456/t8589942982(8589942982) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826318 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:3.0:1787826300.053890:0:67227:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1787826300.053901:0:67227:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826300.053919:0:67227:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826300.053936:0:67227:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:3.0:1787826300.053950:0:67227:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:3.0:1787826300.053956:0:67227:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826300.053963:0:67227:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826300.053966:0:67227:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:3.0:1787826300.053973:0:67227:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141d13cb8 00010000:00010000:3.0:1787826300.053977:0:67227:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:3.0:1787826300.053991:0:67227:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:3.0:1787826300.054095:0:67227:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826300.054103:0:67227:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826300.054111:0:67227:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:3.0:1787826300.054121:0:67227:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:3.0:1787826300.054128:0:67227:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826300.054131:0:67227:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826300.054134:0:67227:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:3.0:1787826300.054138:0:67227:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141d13bb0 00010000:00010000:3.0:1787826300.054141:0:67227:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:3.0:1787826300.054148:0:67227:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:3.0:1787826300.054158:0:67227:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:3.0:1787826300.054164:0:67227:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:3.0:1787826300.054180:0:67227:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7fff7e47c99c 00000080:00200008:3.0:1787826300.054192:0:67227:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:1.0:1787826300.057905:0:67231:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826300.057935:0:67231:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826300.057953:0:67231:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1787826300.057968:0:67231:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:1.0:1787826300.057982:0:67231:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826300.057990:0:67231:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826300.057994:0:67231:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:1.0:1787826300.058000:0:67231:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141dc3bf8 00010000:00010000:1.0:1787826300.058005:0:67231:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1787826300.058013:0:67231:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1787826300.058027:0:67231:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826300.058034:0:67231:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=27 00000080:00200000:1.0:1787826300.058039:0:67231:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 1 00000080:00200000:1.0:1787826300.058052:0:67231:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), file ffff907d863c1980, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1787826300.058059:0:67231:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), intent=open 00000002:00010000:1.0:1787826300.058072:0:67231:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x202:0x0], intent: open flags 0304001 00010000:00010000:1.0:1787826300.058124:0:67231:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8ab lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67231 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.058132:0:67231:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8ab lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67231 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826300.058142:0:67231:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826300.058154:0:67231:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0b40 x1874670011428096/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826300.058164:0:67231:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8ab lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67231 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826300.058185:0:67231:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67231:1874670011428096:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826300.058377:0:67231:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d96147000 going to sleep for 13 seconds 00000100:00100000:1.0:1787826300.062469:0:67231:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67231:1874670011428096:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826300.062494:0:67231:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8ab lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67231 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.062507:0:67231:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8ab lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 67231 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.062514:0:67231:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8ab lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 67231 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.062522:0:67231:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8ab lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 67231 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.062530:0:67231:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8ab lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 67231 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.062537:0:67231:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0b400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826300.062541:0:67231:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8ab lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 67231 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.062548:0:67231:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8ab lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 67231 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.062556:0:67231:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8ab lrc: 5/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 67231 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.062563:0:67231:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826300.062570:0:67231:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8ab lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 67231 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.062577:0:67231:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8ab lrc: 0/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 67231 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826300.062596:0:67231:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908a0b40 x1874670011428096/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826318 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826300.062624:0:67231:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x202:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826300.062850:0:67231:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826300.062867:0:67231:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826300.062881:0:67231:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1787826300.062893:0:67231:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1787826300.062900:0:67231:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826300.062905:0:67231:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826300.062909:0:67231:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:1.0:1787826300.062914:0:67231:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141dc3bf8 00010000:00010000:1.0:1787826300.062917:0:67231:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1787826300.062925:0:67231:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1787826300.062938:0:67231:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826300.062946:0:67231:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=267 00000080:00200000:1.0:1787826300.062953:0:67231:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 44 00000080:00200008:1.0:1787826300.062960:0:67231:0:(file.c:1090:ll_file_open()) START file d204c.sanity-flr:[0x200000404:0x202:0x0](ffff907d93313248), flags 304000 00000002:00010000:1.0:1787826300.062975:0:67231:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x202:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240304001 00010000:00010000:1.0:1787826300.063017:0:67231:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b8b2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67231 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.063024:0:67231:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b8b2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67231 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826300.063034:0:67231:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826300.063041:0:67231:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0b40 x1874670011428352/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826300.063051:0:67231:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b8b2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67231 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826300.063070:0:67231:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67231:1874670011428352:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826300.063238:0:67231:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d96147000 going to sleep for 13 seconds 00000100:00100000:1.0:1787826300.069336:0:67231:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67231:1874670011428352:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826300.069368:0:67231:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b8b2 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff76 expref: -99 pid: 67231 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.069380:0:67231:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b8b2 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff76 expref: -99 pid: 67231 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.069392:0:67231:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b8b2 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x202:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ff76 expref: -99 pid: 67231 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.069402:0:67231:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b8b2 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x202:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff76 expref: -99 pid: 67231 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826300.069472:0:67231:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x200002b, status=0 req@ffff907d908a0b40 x1874670011428352/t8589942984(8589942984) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826318 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826300.069487:0:67231:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a0b40 x1874670011428352/t8589942984(8589942984) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826318 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826300.069497:0:67231:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b8b2 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x202:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff76 expref: -99 pid: 67231 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.069507:0:67231:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/514 (0 0) 00000080:00200000:1.0:1787826300.069512:0:67231:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826300.069530:0:67231:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x202:0x0], updating i_size 4096 00000080:00010000:1.0:1787826300.069535:0:67231:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x202:0x0](ffff907d93313248) for lock 0xe5f736940357b8b2 00000080:00010000:1.0:1787826300.069543:0:67231:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b8b2 from it ffff9dd141dc3bd0 00010000:00010000:1.0:1787826300.069547:0:67231:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b8b2 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x202:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0ff76 expref: -99 pid: 67231 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.069556:0:67231:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b8b2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x202:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff76 expref: -99 pid: 67231 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1787826300.069578:0:67231:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a0b40 x1874670011428352/t8589942984(8589942984) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826318 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1787826300.069672:0:67231:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248) cmd=c1406916 arg=22d7a20 00000100:00100000:1.0:1787826300.069769:0:67231:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67231:1874670011428480:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826300.069840:0:67231:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d96147000 going to sleep for 13 seconds 00000100:00100000:1.0:1787826300.073740:0:67231:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67231:1874670011428480:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826300.073891:0:67231:0:(file.c:479:ll_file_release()) START file d204c.sanity-flr:[0x200000404:0x202:0x0](ffff907d93313248), flags 304000 00010000:00010000:1.0:1787826300.073916:0:67231:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b8b2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x202:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff76 expref: -99 pid: 67231 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200008:3.0:1787826300.086557:0:67234:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1787826300.086568:0:67234:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826300.086586:0:67234:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826300.086601:0:67234:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1787826300.087337:0:67234:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:2.0:1787826300.087348:0:67234:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826300.087356:0:67234:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826300.087360:0:67234:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:2.0:1787826300.087369:0:67234:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141d9bcb8 00010000:00010000:2.0:1787826300.087373:0:67234:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1787826300.087382:0:67234:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:2.0:1787826300.087505:0:67234:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826300.087515:0:67234:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826300.087524:0:67234:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1787826300.087533:0:67234:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:2.0:1787826300.087540:0:67234:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826300.087544:0:67234:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826300.087546:0:67234:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:2.0:1787826300.087551:0:67234:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141d9bbb0 00010000:00010000:2.0:1787826300.087554:0:67234:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1787826300.087561:0:67234:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:2.0:1787826300.087580:0:67234:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:2.0:1787826300.087588:0:67234:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:2.0:1787826300.087625:0:67234:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffd72c03dfc 00000080:00200008:2.0:1787826300.087639:0:67234:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:1.0:1787826300.088757:0:67238:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826300.088773:0:67238:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826300.088785:0:67238:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1787826300.088798:0:67238:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:1.0:1787826300.088803:0:67238:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826300.088807:0:67238:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826300.088810:0:67238:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:1.0:1787826300.088815:0:67238:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141f53bf8 00010000:00010000:1.0:1787826300.088817:0:67238:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:1.0:1787826300.088824:0:67238:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:1.0:1787826300.088834:0:67238:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826300.088839:0:67238:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=27 00000080:00200000:1.0:1787826300.088844:0:67238:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 1 00000080:00200000:1.0:1787826300.088854:0:67238:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), file ffff907d863c1800, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1787826300.088859:0:67238:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), intent=open 00000002:00010000:1.0:1787826300.088871:0:67238:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x202:0x0], intent: open flags 0304001 00010000:00010000:1.0:1787826300.088928:0:67238:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8b9 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 67238 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.088937:0:67238:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8b9 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67238 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826300.088950:0:67238:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826300.088964:0:67238:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a03c0 x1874670011428736/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826300.088975:0:67238:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8b9 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67238 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826300.088996:0:67238:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67238:1874670011428736:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826300.089193:0:67238:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d96147000 going to sleep for 13 seconds 00000100:00100000:1.0:1787826300.092734:0:67238:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67238:1874670011428736:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826300.092757:0:67238:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8b9 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67238 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.092770:0:67238:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8b9 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 67238 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.092777:0:67238:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8b9 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 67238 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.092785:0:67238:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8b9 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 67238 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.092793:0:67238:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8b9 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 67238 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.092799:0:67238:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0b400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826300.092803:0:67238:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8b9 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 67238 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.092810:0:67238:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8b9 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 67238 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.092817:0:67238:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8b9 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 67238 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.092825:0:67238:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826300.092831:0:67238:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8b9 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 67238 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.092838:0:67238:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b8b9 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 67238 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826300.092856:0:67238:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908a03c0 x1874670011428736/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826318 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826300.092868:0:67238:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x202:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826300.093091:0:67238:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826300.093111:0:67238:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826300.093125:0:67238:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1787826300.093137:0:67238:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:1.0:1787826300.093144:0:67238:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826300.093149:0:67238:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826300.093153:0:67238:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:1.0:1787826300.093158:0:67238:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141f53bf8 00010000:00010000:1.0:1787826300.093162:0:67238:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1787826300.093169:0:67238:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:1.0:1787826300.093183:0:67238:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826300.093190:0:67238:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=267 00000080:00200000:1.0:1787826300.093196:0:67238:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 44 00000080:00200008:1.0:1787826300.093204:0:67238:0:(file.c:1090:ll_file_open()) START file d204c.sanity-flr:[0x200000404:0x202:0x0](ffff907d93313248), flags 304000 00000080:00200080:1.0:1787826300.093236:0:67238:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248) cmd=c1406916 arg=197aa20 00000100:00100000:1.0:1787826300.093299:0:67238:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67238:1874670011428864:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826300.093376:0:67238:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d96147000 going to sleep for 13 seconds 00000100:00100000:1.0:1787826300.099027:0:67238:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67238:1874670011428864:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826300.099185:0:67238:0:(file.c:479:ll_file_release()) START file d204c.sanity-flr:[0x200000404:0x202:0x0](ffff907d93313248), flags 304000 00010000:00010000:1.0:1787826300.099213:0:67238:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b8b2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x202:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff76 expref: -99 pid: 67231 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1787826300.131495:0:67241:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826300.131524:0:67241:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826300.131543:0:67241:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:3.0:1787826300.131560:0:67241:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:3.0:1787826300.131569:0:67241:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826300.131577:0:67241:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826300.131581:0:67241:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:3.0:1787826300.131588:0:67241:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141f63bf8 00010000:00010000:3.0:1787826300.131592:0:67241:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:3.0:1787826300.131602:0:67241:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:3.0:1787826300.131650:0:67241:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826300.131659:0:67241:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=27 00000080:00200000:3.0:1787826300.131665:0:67241:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 1 00000080:00200000:3.0:1787826300.131677:0:67241:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), file ffff907d83cc2300, open_flags 18800, mode 0 opened 0 00000080:00200000:3.0:1787826300.131684:0:67241:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), intent=open 00000002:00010000:3.0:1787826300.131699:0:67241:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x202:0x0], intent: open flags 0304001 00010000:00010000:3.0:1787826300.131752:0:67241:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef200/0xe5f736940357b8c0 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 67241 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826300.131761:0:67241:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef200/0xe5f736940357b8c0 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67241 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826300.131773:0:67241:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826300.131785:0:67241:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d835243c0 x1874670011429120/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826300.131796:0:67241:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef200/0xe5f736940357b8c0 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67241 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826300.131817:0:67241:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835243c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67241:1874670011429120:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826300.135395:0:67241:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bba80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826300.140723:0:67241:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835243c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67241:1874670011429120:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826300.140747:0:67241:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef200/0xe5f736940357b8c0 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67241 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.140761:0:67241:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef200/0xe5f736940357b8c0 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 67241 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.140769:0:67241:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef200/0xe5f736940357b8c0 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 67241 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.140777:0:67241:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef200/0xe5f736940357b8c0 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 67241 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.140785:0:67241:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef200/0xe5f736940357b8c0 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 67241 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.140792:0:67241:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907da59ef200 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826300.140797:0:67241:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef200/0xe5f736940357b8c0 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 67241 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.140805:0:67241:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef200/0xe5f736940357b8c0 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 67241 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.140813:0:67241:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef200/0xe5f736940357b8c0 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 67241 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.140820:0:67241:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826300.140827:0:67241:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef200/0xe5f736940357b8c0 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 67241 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.140834:0:67241:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ef200/0xe5f736940357b8c0 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 67241 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826300.140852:0:67241:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d835243c0 x1874670011429120/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826318 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826300.140866:0:67241:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x202:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1787826300.140994:0:67241:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826300.141013:0:67241:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826300.141028:0:67241:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:0.0:1787826300.141040:0:67241:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:0.0:1787826300.141048:0:67241:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826300.141055:0:67241:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826300.141058:0:67241:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:0.0:1787826300.141064:0:67241:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141f63bf8 00010000:00010000:0.0:1787826300.141069:0:67241:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:0.0:1787826300.141077:0:67241:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:0.0:1787826300.141092:0:67241:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826300.141099:0:67241:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=267 00000080:00200000:0.0:1787826300.141105:0:67241:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 44 00000080:00200008:0.0:1787826300.141113:0:67241:0:(file.c:1090:ll_file_open()) START file d204c.sanity-flr:[0x200000404:0x202:0x0](ffff907d93313248), flags 304000 00000080:00200080:0.0:1787826300.141146:0:67241:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248) cmd=c0086915 arg=e3d3d0 00000100:00100000:0.0:1787826300.141198:0:67241:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67241:1874670011429248:192.168.203.104@tcp:34:lfs.0 00000100:00100000:0.0:1787826300.141277:0:67241:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f916c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826300.146710:0:67241:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67241:1874670011429248:192.168.203.104@tcp:34:lfs.0 00000080:00200008:0.0:1787826300.146895:0:67241:0:(file.c:479:ll_file_release()) START file d204c.sanity-flr:[0x200000404:0x202:0x0](ffff907d93313248), flags 304000 00010000:00010000:0.0:1787826300.146924:0:67241:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b8b2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x202:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff76 expref: -99 pid: 67231 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1787826300.168510:0:67244:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826300.168540:0:67244:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826300.168559:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826300.168575:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:3.0:1787826300.168582:0:67244:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826300.173678:0:67244:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826300.173688:0:67244:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:3.0:1787826300.173699:0:67244:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141f63b58 00010000:00010000:3.0:1787826300.173706:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826300.173721:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:3.0:1787826300.173740:0:67244:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826300.173751:0:67244:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=25 00000080:00200000:3.0:1787826300.173759:0:67244:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 1 00000080:00200000:3.0:1787826300.173774:0:67244:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), flags=9 00000080:00200000:3.0:1787826300.173778:0:67244:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), intent=getattr 00000002:00010000:3.0:1787826300.173797:0:67244:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x202:0x0], intent: getattr flags 01 00010000:00010000:3.0:1787826300.173848:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 67244 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826300.173857:0:67244:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67244 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826300.173865:0:67244:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67244 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826300.173889:0:67244:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011429376:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826300.173990:0:67244:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb9c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826300.190696:0:67244:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011429376:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826300.190722:0:67244:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.190740:0:67244:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.190749:0:67244:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.190762:0:67244:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.190773:0:67244:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.190787:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.190794:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.190801:0:67244:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826300.190814:0:67244:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d83524000 x1874670011429376/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/1032 e 0 to 0 dl 1787826318 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826300.190825:0:67244:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 448 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1787826300.190841:0:67244:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.190855:0:67244:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/515 (0 0) 00000080:00200000:0.0:1787826300.190863:0:67244:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x202:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1787826300.190914:0:67244:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x203:0x0](ffff907d93310908): rc = 0 00000080:00200000:0.0:1787826300.190930:0:67244:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x203:0x0]: layout version change: 7 -> 7 00000080:00010000:0.0:1787826300.190935:0:67244:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) for lock 0xe5f736940357b8c7 00000080:00010000:0.0:1787826300.190946:0:67244:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) 00000080:00010000:0.0:1787826300.190950:0:67244:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b8c7 from it ffff9dd141f63b20 00010000:00010000:0.0:1787826300.190954:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.190969:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:0.0:1787826300.191004:0:67244:0:(file.c:6313:ll_getattr_dentry()) START file f204c.sanity-flr:[0x200000404:0x203:0x0](ffff907d93310908), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1787826300.191010:0:67244:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908),name=f204c.sanity-flr 00000002:00010000:0.0:1787826300.191028:0:67244:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x203:0x0]) in obj [0x200000404:0x202:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826300.191038:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826300.191048:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826300.191055:0:67244:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x202:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826300.191060:0:67244:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) 00000080:00010000:0.0:1787826300.191063:0:67244:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) for lock 0xe5f736940357b8c7 00000080:00010000:0.0:1787826300.191066:0:67244:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b8c7 from it ffff9dd141f63cb8 00010000:00010000:0.0:1787826300.191070:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826300.191077:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826300.191096:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:0.0:1787826300.191107:0:67244:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826300.191111:0:67244:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:0.0:1787826300.191115:0:67244:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x203:0x0] 00000008:00010000:0.0:1787826300.191131:0:67244:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1787826300.191134:0:67244:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da032e1b0, osc lock ffff907d96369248, flags 0x400000400001002 00010000:00010000:0.0:1787826300.191179:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 3/1,0 mode: --/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.191189:0:67244:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 3/1,0 mode: --/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.191201:0:67244:0:(ldlm_request.c:1139:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff907d90844f00 x1874670011429504/t0(0) o101->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000020:00200000:0.0:1787826300.191249:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:0.0:1787826300.191263:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1787826300.191279:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011429504:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826300.199754:0:2349:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 6/2,0 mode: --/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.199780:0:2349:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 6/2,0 mode: --/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10009 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.199799:0:2349:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10009 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.199810:0:2349:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10009 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826300.199824:0:2349:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10009 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1787826300.199835:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1787826300.199855:0:2349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10009 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.199864:0:2349:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10009 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.199872:0:2349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10009 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.199881:0:2349:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10009 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1787826300.199901:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011429504:192.168.203.104@tcp:101:lfs.0 00000080:00200000:3.0:1787826300.199950:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:3.0:1787826300.201259:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5793160 (ffff907d884a7c68/0) at cl_lock_release():245 00000020:00010000:3.0:1787826300.201266:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5793160 (ffff907d884a7c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826300.201274:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da032e1b0 (ffff907d884a7008/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826300.201292:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5793160 (ffff907d884a7c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826300.201296:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da032e1b0 (ffff907d884a7c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826300.201321:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787826300.201404:0:67244:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826300.201433:0:67244:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826300.201457:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:3.0:1787826300.201476:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:3.0:1787826300.201486:0:67244:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826300.201494:0:67244:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826300.201499:0:67244:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:3.0:1787826300.201508:0:67244:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141f63bf8 00010000:00010000:3.0:1787826300.201512:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:3.0:1787826300.201522:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:3.0:1787826300.201539:0:67244:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826300.201547:0:67244:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=25 00000080:00200000:3.0:1787826300.201553:0:67244:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 1 00000080:00200000:3.0:1787826300.201558:0:67244:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204c.sanity-flr, flags=265 00000080:00200000:3.0:1787826300.201563:0:67244:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908), inode mode 81a4 mask 46 00000080:00200008:3.0:1787826300.201572:0:67244:0:(file.c:1090:ll_file_open()) START file f204c.sanity-flr:[0x200000404:0x203:0x0](ffff907d93310908), flags 154402 00000002:00010000:3.0:1787826300.201589:0:67244:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x203:0x0]) in obj [0x200000404:0x202:0x0], intent: open flags 0100240114403 00010000:00010000:3.0:1787826300.201680:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 67244 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826300.201694:0:67244:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67244 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826300.201708:0:67244:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826300.201723:0:67244:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83527840 x1874670011429760/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826300.201734:0:67244:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67244 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826300.201763:0:67244:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011429760:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826300.202221:0:67244:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb9c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826300.209209:0:67244:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011429760:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826300.209246:0:67244:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10017 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.209265:0:67244:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10017 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.209273:0:67244:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10017 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.209296:0:67244:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10017 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.209310:0:67244:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10017 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.209326:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10017 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.209335:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10017 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.209343:0:67244:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10017 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826300.209357:0:67244:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffff907d83527840 x1874670011429760/t8589942985(8589942985) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1056 e 0 to 0 dl 1787826318 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1787826300.209372:0:67244:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d83527840 x1874670011429760/t8589942985(8589942985) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1056 e 0 to 0 dl 1787826318 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826300.209387:0:67244:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 448 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10017 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1787826300.209405:0:67244:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10017 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.209419:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826300.209433:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826300.209441:0:67244:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10017 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.209447:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x7918213f4fc10017 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.209454:0:67244:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x7918213f4fc10017 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.209494:0:67244:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:0.0:1787826300.209503:0:67244:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x202:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1787826300.209580:0:67244:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x203:0x0]: layout version change: 7 -> 7 00000080:00010000:0.0:1787826300.209586:0:67244:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) for lock 0xe5f736940357b8c7 00000080:00010000:0.0:1787826300.209594:0:67244:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b8c7 from it ffff9dd141f63bf0 00010000:00010000:0.0:1787826300.209598:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826300.209662:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000002:00100000:0.0:1787826300.209692:0:67244:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d83527840 x1874670011429760/t8589942985(8589942985) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 968/1056 e 0 to 0 dl 1787826318 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826300.209881:0:67244:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908), inode mode 81a4 mask 4 00000080:00200000:0.0:1787826300.209892:0:67244:0:(xattr.c:760:ll_xattr_get()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908), xattr lov 00000080:00200008:0.0:1787826300.209922:0:67244:0:(file.c:6313:ll_getattr_dentry()) START file f204c.sanity-flr:[0x200000404:0x203:0x0](ffff907d93310908), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1787826300.209929:0:67244:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908),name=f204c.sanity-flr 00000002:00010000:0.0:1787826300.209949:0:67244:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x203:0x0]) in obj [0x200000404:0x202:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826300.210030:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826300.210042:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1787826300.210049:0:67244:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x202:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826300.210056:0:67244:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) 00000080:00010000:0.0:1787826300.210060:0:67244:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) for lock 0xe5f736940357b8c7 00000080:00010000:0.0:1787826300.210064:0:67244:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b8c7 from it ffff9dd141f63cd8 00010000:00010000:0.0:1787826300.210067:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826300.210075:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1787826300.210103:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:0.0:1787826300.210117:0:67244:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826300.210121:0:67244:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:0.0:1787826300.210126:0:67244:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x203:0x0] 00000008:00010000:0.0:1787826300.210174:0:67244:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1787826300.210178:0:67244:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da032eeb0, osc lock ffff907d96369728, flags 0x400000400001002 00010000:00010000:0.0:1787826300.210197:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10009 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826300.210210:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10009 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:0.0:1787826300.210245:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1787826300.210250:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10009 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826300.210258:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10009 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:0.0:1787826300.210268:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1787826300.210278:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:0.0:1787826300.210292:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da57918b0 (ffff907d884a76c8/0) at cl_lock_release():245 00000020:00010000:0.0:1787826300.210296:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da57918b0 (ffff907d884a76c8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1787826300.210299:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da032eeb0 (ffff907d884a7248/1) at cl_lock_cancel():142 00000020:00010000:0.0:1787826300.210304:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da57918b0 (ffff907d884a76c8/0) at cl_lock_fini():72 00000020:00010000:0.0:1787826300.210306:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da032eeb0 (ffff907d884a76c8/1) at cl_lock_fini():72 00000080:00200000:0.0:1787826300.210319:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:0.0:1787826300.210367:0:67244:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908) cmd=c00c66f3 arg=16c12a0 00000002:00010000:0.0:1787826300.210387:0:67244:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x203:0x0]) in obj [0x200000404:0x203:0x0], intent: open flags 05004040000002 00010000:00010000:0.0:1787826300.210446:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b8dc lrc: 3/1,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x10000000000000 pid: 67244 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.210455:0:67244:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x81000000001000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b8dc lrc: 3/1,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0x1/0x0 rrc: 6 type: IBT flags: 0x0 pid: 67244 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826300.210467:0:67244:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826300.210476:0:67244:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90844f00 x1874670011430016/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826300.210487:0:67244:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b8dc lrc: 3/1,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0x1/0x0 rrc: 6 type: IBT flags: 0x81000000000000 pid: 67244 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826300.210514:0:67244:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011430016:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826300.210808:0:67244:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f913c0 going to sleep for 13 seconds 00010000:00010000:2.0:1787826300.210920:0:47852:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1787826300.210930:0:47852:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7918213f4fc10017 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.210940:0:47852:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907da59ee200 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:2.0:1787826300.210946:0:47852:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x1b rrc: 6 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7918213f4fc10017 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.210955:0:47852:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x1b rrc: 6 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7918213f4fc10017 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.210975:0:47852:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 3/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x1b rrc: 6 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc10017 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.210996:0:47852:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826300.211031:0:47852:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 2/0,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x1b rrc: 6 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc10017 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.211039:0:47852:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1787826300.211067:0:47852:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x1b rrc: 6 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc10017 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.211075:0:47852:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee200/0xe5f736940357b8d5 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x1b rrc: 6 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc10017 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.211098:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826300.211099:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826300.211101:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826300.211116:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011430144:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826300.219788:0:67244:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011430016:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826300.219825:0:67244:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PW ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b8dc lrc: 4/1,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc1002c expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.219841:0:67244:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b8dc lrc: 4/1,0 mode: --/PW res: [0x200000404:0x203:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc1002c expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.219856:0:67244:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b8dc lrc: 4/1,0 mode: PW/PW res: [0x200000404:0x203:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc1002c expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.219877:0:67244:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b8dc lrc: 4/1,0 mode: PW/PW res: [0x200000404:0x203:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc1002c expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.219890:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b8dc lrc: 5/1,1 mode: PW/PW res: [0x200000404:0x203:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc1002c expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.219898:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b8dc lrc: 5/1,1 mode: PW/PW res: [0x200000404:0x203:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc1002c expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.219905:0:67244:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b8dc lrc: 4/0,1 mode: PW/PW res: [0x200000404:0x203:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc1002c expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826300.219918:0:67244:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x600002b, status=0 req@ffff907d90844f00 x1874670011430016/t8589942986(8589942986) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1056 e 0 to 0 dl 1787826318 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1787826300.219930:0:67244:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90844f00 x1874670011430016/t8589942986(8589942986) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1056 e 0 to 0 dl 1787826318 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826300.219946:0:67244:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b8dc lrc: 3/0,1 mode: PW/PW res: [0x200000404:0x203:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc1002c expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.219962:0:67244:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 2 res 8589935620/515 (0 0) 00000080:00200000:0.0:1787826300.219970:0:67244:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x203:0x0] suppgids 0 0: rc 0 00000002:00100000:0.0:1787826300.219991:0:67244:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90844f00 x1874670011430016/t8589942986(8589942986) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 968/1056 e 0 to 0 dl 1787826318 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:0.0:1787826300.220000:0:67244:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) for lock 0xe5f736940357b8dc 00000080:00010000:0.0:1787826300.220008:0:67244:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b8dc from it ffff9dd141f63cc0 00010000:00010000:0.0:1787826300.220012:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b8dc lrc: 3/0,1 mode: PW/PW res: [0x200000404:0x203:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc1002c expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.220019:0:67244:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b8dc lrc: 2/0,0 mode: PW/PW res: [0x200000404:0x203:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc1002c expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826300.220081:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:0.0:1787826300.220098:0:67244:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826300.220101:0:67244:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826300.220109:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826300.220121:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000100:00100000:1.0:1787826300.220313:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011430272:192.168.203.104@tcp:1:lfs.0 00000100:00100000:2.0:1787826300.220654:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011430144:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1787826300.225005:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826300.228954:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025494784 00000100:00100000:2.0:1787826300.228975:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025494784 00000100:00100000:2.0:1787826300.229100:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7146 00000100:00100000:2.0:1787826300.229112:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83527c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:35506:x1874670025494784:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826300.229136:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc10009 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826300.229254:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826300.229261:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83527c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:35506:x1874670025494784:12345-192.168.203.104@tcp:104: Request processed in 152us (4271us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826300.229269:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7146 00010000:00010000:1.0:1787826300.229401:0:40346:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826300.229416:0:40346:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc10009 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826300.229439:0:40346:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883aa00 already unused, calling callback (ffffffffc118e410) 00010000:00010000:1.0:1787826300.229447:0:40346:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800428400000000 nid: local remote: 0x7918213f4fc10009 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826300.229469:0:40346:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1787826300.229482:0:40346:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1787826300.229536:0:40346:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 2/0,0 mode: --/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc10009 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826300.229544:0:40346:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826300.229574:0:40346:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 1/0,0 mode: --/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc10009 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826300.229583:0:40346:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000bb52c60e/0xe5f736940357b8ce lrc: 0/0,0 mode: --/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc10009 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826300.229980:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826300.229984:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826300.229987:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826300.230018:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011430400:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826300.239685:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011430400:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826300.240164:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011430272:192.168.203.104@tcp:1:lfs.0 00010000:00010000:0.0:1787826300.240350:0:67244:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:0.0:1787826300.240379:0:67244:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x203:0x0]: losing layout lock 00010000:00010000:0.0:1787826300.240399:0:67244:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1787826300.240408:0:67244:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x203:0x0]: losing layout lock 00010000:00010000:0.0:1787826300.240439:0:67244:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1787826300.240447:0:67244:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 1/0,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826300.240453:0:67244:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1787826300.240457:0:67244:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907da59ee600/0xe5f736940357b8c7 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc10002 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1787826300.240470:0:67244:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b88f lrc: 0/0,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc0ff06 expref: -99 pid: 67203 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000020:00100000:0.0:1787826300.240484:0:67244:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1787826300.240492:0:67244:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90844b40 x1874670011430656/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 472/440 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787826300.240513:0:67244:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011430656:192.168.203.104@tcp:36:lfs.0 00000100:00100000:0.0:1787826300.240590:0:67244:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f913c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826300.260711:0:67244:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589942987 gen 1 00000100:00100000:0.0:1787826300.260731:0:67244:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d908a0b40 x1874670011428352/t8589942984(8589942984) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826318 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1787826300.260744:0:67244:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d83527840 x1874670011429760/t8589942985(8589942985) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 968/1056 e 0 to 0 dl 1787826318 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1787826300.260752:0:67244:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d90844f00 x1874670011430016/t8589942986(8589942986) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 968/1056 e 0 to 0 dl 1787826318 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1787826300.260763:0:67244:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011430656:192.168.203.104@tcp:36:lfs.0 00010000:00010000:0.0:1787826300.260806:0:67244:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 31 res 8589935620/515 (0 0) 00000080:00010000:0.0:1787826300.260819:0:67244:0:(file.c:7076:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000404:0x203:0x0](ffff907d93310908) 00000002:00010000:0.0:1787826300.260830:0:67244:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x203:0x0]) in obj [0x200000404:0x203:0x0], intent: layout flags 0 00010000:00010000:0.0:1787826300.260873:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 67244 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.260881:0:67244:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67244 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.260891:0:67244:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67244 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826300.260911:0:67244:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011430784:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826300.261168:0:67244:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f913c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826300.263494:0:67244:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011430784:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826300.263536:0:67244:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.263554:0:67244:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826300.263572:0:67244:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x400 disposition=0x1, status=0 req@ffff907d90844b40 x1874670011430784/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/792 e 0 to 0 dl 1787826318 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826300.263586:0:67244:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 448 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1787826300.263601:0:67244:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.263650:0:67244:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/515 (0 0) 00000080:00200000:0.0:1787826300.263658:0:67244:0:(namei.c:952:ll_intent_lock()) intent lock 1024 on i1 [0x200000404:0x203:0x0] suppgids 0 0: rc 0 00000080:00010000:0.0:1787826300.263679:0:67244:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) for lock 0xe5f736940357b8e3 00000080:00010000:0.0:1787826300.263689:0:67244:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b8e3 from it ffff9dd141f63c70 00010000:00010000:0.0:1787826300.263694:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.263712:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826300.263924:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826300.263940:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:0.0:1787826300.263950:0:67244:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x203:0x0](00000000010d8bed) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826300.264008:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826300.264017:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826300.264037:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826300.264050:0:67244:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826300.264054:0:67244:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826300.264063:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826300.264186:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:0.0:1787826300.264212:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1787826300.264227:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826300.264241:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1787826300.264249:0:67244:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x203:0x0](00000000010d8bed) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826300.264267:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826300.264271:0:67244:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826300.264274:0:67244:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826300.264277:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826300.264309:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:0.0:1787826300.264317:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1787826300.264397:0:67244:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x203:0x0]: layout version change: 4294967294 -> 8 00010000:00010000:0.0:1787826300.264403:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826300.264413:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200080:0.0:1787826300.264442:0:67244:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908) cmd=66f4 arg=ffffffffffffffff 00000080:00200008:0.0:1787826300.264458:0:67244:0:(file.c:6313:ll_getattr_dentry()) START file f204c.sanity-flr:[0x200000404:0x203:0x0](ffff907d93310908), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1787826300.264466:0:67244:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908),name=f204c.sanity-flr 00000002:00010000:0.0:1787826300.264482:0:67244:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x203:0x0]) in obj [0x200000404:0x203:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826300.264490:0:67244:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 30 res 8589935620/515 (0 0) 00010000:00010000:0.0:1787826300.264535:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 3/1,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 67244 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.264547:0:67244:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 3/1,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 67244 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.264557:0:67244:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 3/1,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 67244 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826300.264584:0:67244:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011430912:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826300.264953:0:67244:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f913c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826300.269277:0:67244:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011430912:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826300.269328:0:67244:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 4/1,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.269344:0:67244:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 4/1,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.269358:0:67244:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.269371:0:67244:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.269386:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.269395:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.269408:0:67244:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826300.269421:0:67244:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d90844b40 x1874670011430912/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/1032 e 0 to 0 dl 1787826318 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826300.269434:0:67244:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 448 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1787826300.269451:0:67244:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.269465:0:67244:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/515 (0 0) 00000080:00200000:0.0:1787826300.269473:0:67244:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x203:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1787826300.269548:0:67244:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x203:0x0]: layout version change: 8 -> 8 00000080:00010000:0.0:1787826300.269555:0:67244:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) for lock 0xe5f736940357b8ea 00000080:00010000:0.0:1787826300.269561:0:67244:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) 00000080:00010000:0.0:1787826300.269565:0:67244:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b8ea from it ffff9dd141f63cd8 00010000:00010000:0.0:1787826300.269568:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.269577:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826300.269658:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:0.0:1787826300.269671:0:67244:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826300.269675:0:67244:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:0.0:1787826300.269679:0:67244:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x203:0x0] 00000008:00010000:0.0:1787826300.269701:0:67244:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1787826300.269705:0:67244:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da032e530, osc lock ffff907d96369728, flags 0x400000400001002 00010000:00010000:0.0:1787826300.269765:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.269777:0:67244:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.269788:0:67244:0:(ldlm_request.c:1139:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff907d90844b40 x1874670011431040/t0(0) o101->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000020:00200000:0.0:1787826300.269837:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:0.0:1787826300.269852:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1787826300.269874:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011431040:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826300.274589:0:2348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.274666:0:2348:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.274685:0:2348:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.274698:0:2348:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826300.274717:0:2348:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1787826300.274749:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1787826300.274771:0:2348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.274781:0:2348:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.274788:0:2348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.274799:0:2348:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1787826300.274825:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011431040:192.168.203.104@tcp:101:lfs.0 00000080:00200000:3.0:1787826300.274923:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:3.0:1787826300.274931:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da57918b0 (ffff907d884a76c8/0) at cl_lock_release():245 00000020:00010000:3.0:1787826300.274934:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da57918b0 (ffff907d884a76c8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826300.274939:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da032e530 (ffff907d884a7248/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826300.274944:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da57918b0 (ffff907d884a76c8/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826300.274947:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da032e530 (ffff907d884a76c8/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826300.274962:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/1, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:2.0:1787826300.328844:0:67244:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908) cmd=400866b1 arg=1 00000080:00200008:2.0:1787826300.328872:0:67244:0:(file.c:6313:ll_getattr_dentry()) START file f204c.sanity-flr:[0x200000404:0x203:0x0](ffff907d93310908), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1787826300.328884:0:67244:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908),name=f204c.sanity-flr 00000002:00010000:2.0:1787826300.328913:0:67244:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x203:0x0]) in obj [0x200000404:0x203:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826300.328940:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826300.328960:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826300.328971:0:67244:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x203:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826300.328984:0:67244:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) 00000080:00010000:2.0:1787826300.328989:0:67244:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) for lock 0xe5f736940357b8ea 00000080:00010000:2.0:1787826300.328999:0:67244:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b8ea from it ffff9dd141f63cd8 00010000:00010000:2.0:1787826300.329004:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826300.329015:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1787826300.329052:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:2.0:1787826300.329068:0:67244:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826300.329071:0:67244:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1787826300.329076:0:67244:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x203:0x0] 00000008:00010000:2.0:1787826300.329185:0:67244:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826300.329190:0:67244:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322030, osc lock ffff907d830a29c0, flags 0x400000400001002 00010000:00010000:2.0:1787826300.329214:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826300.329226:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1787826300.329241:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826300.329245:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826300.329260:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1787826300.329271:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826300.329294:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826300.329300:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_release():245 00000020:00010000:2.0:1787826300.329303:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826300.329306:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322030 (ffff907da5608878/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826300.329312:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5790000 (ffff907d884a7f38/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826300.329314:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322030 (ffff907d884a7f38/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826300.329329:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/1, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826300.329352:0:67244:0:(file.c:5224:ll_file_seek()) START file f204c.sanity-flr:[0x200000404:0x203:0x0], offset: 0, type: SEEK_DATA 00000080:00200000:2.0:1787826300.329358:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 8 restore needed 0 00020000:00200000:2.0:1787826300.329362:0:67244:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826300.329364:0:67244:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1787826300.329370:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1787826300.329375:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:2.0:1787826300.329393:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000080:00200000:2.0:1787826300.329402:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826300.329408:0:67244:0:(file.c:1848:ll_io_set_mirror()) f204c.sanity-flr: desiginated mirror: 1 00000080:00200000:2.0:1787826300.329411:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1787826300.329417:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1787826300.329420:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:2.0:1787826300.329423:0:67244:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 18446744073709551615] 00000008:00010000:2.0:1787826300.329429:0:67244:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:2.0:1787826300.329432:0:67244:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322030, osc lock ffff907d830a29c0, flags 0x0 00010000:00010000:2.0:1787826300.329441:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826300.329452:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826300.329462:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826300.329469:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826300.329476:0:67244:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1787826300.329487:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000100:00100000:3.0:1787826300.329669:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011431168:192.168.203.104@tcp:23:lfs.0 00000100:00100000:3.0:1787826300.338260:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011431168:192.168.203.104@tcp:23:lfs.0 00000020:00010000:2.0:1787826300.338340:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_release():245 00000020:00010000:2.0:1787826300.338346:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826300.338353:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322030 (ffff907da5608878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1787826300.338367:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826300.338388:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00010000:2.0:1787826300.338408:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826300.338411:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322030 (ffff907d884a7f38/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826300.338433:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826300.338445:0:67244:0:(file.c:5262:ll_file_seek()) COMPLETED file f204c.sanity-flr:[0x200000404:0x203:0x0], offset: 0, type: SEEK_DATA, rc = 0 00000080:00200008:2.0:1787826300.338463:0:67244:0:(file.c:5224:ll_file_seek()) START file f204c.sanity-flr:[0x200000404:0x203:0x0], offset: 0, type: SEEK_HOLE 00000080:00200000:2.0:1787826300.338471:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 8 restore needed 0 00020000:00200000:2.0:1787826300.338481:0:67244:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826300.338484:0:67244:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1787826300.338491:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1787826300.338498:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:2.0:1787826300.338519:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000080:00200000:2.0:1787826300.338524:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826300.338530:0:67244:0:(file.c:1848:ll_io_set_mirror()) f204c.sanity-flr: desiginated mirror: 1 00000080:00200000:2.0:1787826300.338532:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1787826300.338538:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1787826300.338540:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:2.0:1787826300.338543:0:67244:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 18446744073709551615] 00000008:00010000:2.0:1787826300.338558:0:67244:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:2.0:1787826300.338561:0:67244:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322030, osc lock ffff907d830a29c0, flags 0x0 00010000:00010000:2.0:1787826300.338587:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826300.338601:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826300.339169:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826300.339180:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826300.339187:0:67244:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1787826300.339205:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000100:00100000:3.0:1787826300.339838:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011431296:192.168.203.104@tcp:23:lfs.0 00000100:00100000:3.0:1787826300.344534:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011431296:192.168.203.104@tcp:23:lfs.0 00000020:00010000:2.0:1787826300.346665:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_release():245 00000020:00010000:2.0:1787826300.346680:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826300.346687:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322030 (ffff907da5608878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1787826300.346702:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826300.346733:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:2.0:1787826300.346916:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b15d8 (ffff907d884a7f38/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826300.346921:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322030 (ffff907d884a7f38/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826300.346949:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826300.346961:0:67244:0:(file.c:5262:ll_file_seek()) COMPLETED file f204c.sanity-flr:[0x200000404:0x203:0x0], offset: 0, type: SEEK_HOLE, rc = 2097152 00000080:00200080:0.0:1787826300.347164:0:67244:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908) cmd=400866b1 arg=0 00000080:00200080:0.0:1787826300.347180:0:67244:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908) cmd=400866b1 arg=1 00000080:00200008:0.0:1787826300.347189:0:67244:0:(file.c:6313:ll_getattr_dentry()) START file f204c.sanity-flr:[0x200000404:0x203:0x0](ffff907d93310908), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1787826300.347198:0:67244:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908),name=f204c.sanity-flr 00000002:00010000:0.0:1787826300.347226:0:67244:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x203:0x0]) in obj [0x200000404:0x203:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826300.347249:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826300.347264:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826300.347277:0:67244:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x203:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826300.347296:0:67244:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) 00000080:00010000:0.0:1787826300.347300:0:67244:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) for lock 0xe5f736940357b8ea 00000080:00010000:0.0:1787826300.347307:0:67244:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b8ea from it ffff9dd141f63cd8 00010000:00010000:0.0:1787826300.347310:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826300.347318:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1787826300.347345:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:0.0:1787826300.347358:0:67244:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826300.347362:0:67244:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:0.0:1787826300.347367:0:67244:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x203:0x0] 00000008:00010000:0.0:1787826300.347389:0:67244:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1787826300.347395:0:67244:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da032ecb0, osc lock ffff907d96368618, flags 0x400000400001002 00010000:00010000:0.0:1787826300.347413:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1787826300.347424:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:0.0:1787826300.347437:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1787826300.347441:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1787826300.347454:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:0.0:1787826300.347465:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1787826300.347477:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:0.0:1787826300.347481:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5791148 (ffff907d884a7ea8/0) at cl_lock_release():245 00000020:00010000:0.0:1787826300.347483:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5791148 (ffff907d884a7ea8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1787826300.347486:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da032ecb0 (ffff907d884a7a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1787826300.347489:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5791148 (ffff907d884a7ea8/0) at cl_lock_fini():72 00000020:00010000:0.0:1787826300.347492:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da032ecb0 (ffff907d884a7ea8/1) at cl_lock_fini():72 00000080:00200000:0.0:1787826300.347499:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/1, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1787826300.347526:0:67244:0:(file.c:2485:do_file_read_iter()) START file f204c.sanity-flr:[0x200000404:0x203:0x0], ppos: 0, count: 2097152 00000080:00200000:0.0:1787826300.347537:0:67244:0:(file.c:2090:ll_file_io_generic()) f204c.sanity-flr: read ppos: 0, bytes: 2097152 00000080:00200000:0.0:1787826300.347548:0:67244:0:(file.c:1848:ll_io_set_mirror()) f204c.sanity-flr: desiginated mirror: 1 00000020:00200000:0.0:1787826300.347554:0:67244:0:(cl_io.c:218:cl_io_rw_init()) header@000000008ae07a1f[0x110, 2, [0x200000404:0x203:0x0] hash] 00000020:00200000:0.0:1787826300.347556:0:67244:0:(cl_io.c:218:cl_io_rw_init()) io range: 1 [0, 2097152) 2048 0 00000080:00200000:0.0:1787826300.347559:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 8 restore needed 0 00000080:00200000:0.0:1787826300.347570:0:67244:0:(file.c:2170:ll_file_io_generic()) Range lock [0, 511] 00020000:00200000:0.0:1787826300.347579:0:67244:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 2097152) 2097152, 2097152 00020000:00200000:0.0:1787826300.347583:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1787826300.347597:0:67244:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 511] 00000008:00010000:0.0:1787826300.347713:0:67244:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 11 00000008:00010000:0.0:1787826300.347719:0:67244:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da032ecb0, osc lock ffff907d96368618, flags 0x40000 00000080:00200000:0.0:1787826300.347738:0:67244:0:(vvp_io.c:874:vvp_io_read_start()) f204c.sanity-flr: read [0, 2097152) 00000080:00200000:0.0:1787826300.347745:0:67244:0:(vvp_io.c:194:vvp_prep_size()) [0x200000404:0x203:0x0] updating i_size 2097152 00000080:00200000:0.0:1787826300.347952:0:67244:0:(vvp_io.c:932:vvp_io_read_start()) count:2097152 iocb pos:0 00000080:00200000:0.0:1787826300.347962:0:67244:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908), size=2097152 (max 335544320), offset=0=0x0, pages 512 (max 81920), lockless, parallel 00000020:00200000:0.0:1787826300.348891:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 512 items remaining. 00000080:00200000:0.0:1787826300.348898:0:67244:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 2097152 count 2097152 file_offset 2097152 00000020:00010000:0.0:1787826300.348907:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98af23f8 (ffff907d884a7ea8/0) at cl_lock_release():245 00000020:00010000:0.0:1787826300.348910:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98af23f8 (ffff907d884a7ea8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1787826300.348914:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da032ecb0 (ffff907d884a7a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1787826300.348917:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98af23f8 (ffff907d884a7ea8/0) at cl_lock_fini():72 00000020:00010000:0.0:1787826300.348920:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da032ecb0 (ffff907d884a7ea8/1) at cl_lock_fini():72 00000080:00200000:0.0:1787826300.348929:0:67244:0:(vvp_io.c:533:vvp_io_advance()) advancing 2097152 bytes 00000020:00200000:0.0:1787826300.348936:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1787826300.349000:0:64869:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907da70a3c00 00000008:00100000:0.0:1787826300.350039:0:64869:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d90844b40 - object 0x240000400:806 offset 0<>2097152 00000100:00100000:1.0:1787826300.350203:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011431424:192.168.203.104@tcp:3:lfs.0 00000008:00010000:1.0:1787826300.395499:0:2348:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907da70a3c00 00000020:00200000:1.0:1787826300.395563:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 511 items remaining. 00000020:00200000:1.0:1787826300.395568:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 510 items remaining. 00000020:00200000:1.0:1787826300.395571:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 509 items remaining. 00000020:00200000:1.0:1787826300.395573:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 508 items remaining. 00000020:00200000:1.0:1787826300.395575:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 507 items remaining. 00000020:00200000:1.0:1787826300.395577:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 506 items remaining. 00000020:00200000:1.0:1787826300.395578:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 505 items remaining. 00000020:00200000:1.0:1787826300.395580:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 504 items remaining. 00000020:00200000:1.0:1787826300.395582:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 503 items remaining. 00000020:00200000:1.0:1787826300.395583:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 502 items remaining. 00000020:00200000:1.0:1787826300.395585:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 501 items remaining. 00000020:00200000:1.0:1787826300.395586:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 500 items remaining. 00000020:00200000:1.0:1787826300.395588:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 499 items remaining. 00000020:00200000:1.0:1787826300.395589:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 498 items remaining. 00000020:00200000:1.0:1787826300.395591:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 497 items remaining. 00000020:00200000:1.0:1787826300.395592:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 496 items remaining. 00000020:00200000:1.0:1787826300.395593:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 495 items remaining. 00000020:00200000:1.0:1787826300.395594:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 494 items remaining. 00000020:00200000:1.0:1787826300.395596:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 493 items remaining. 00000020:00200000:1.0:1787826300.395597:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 492 items remaining. 00000020:00200000:1.0:1787826300.395599:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 491 items remaining. 00000020:00200000:1.0:1787826300.395600:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 490 items remaining. 00000020:00200000:1.0:1787826300.395601:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 489 items remaining. 00000020:00200000:1.0:1787826300.395602:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 488 items remaining. 00000020:00200000:1.0:1787826300.395604:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 487 items remaining. 00000020:00200000:1.0:1787826300.395633:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 486 items remaining. 00000020:00200000:1.0:1787826300.395635:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 485 items remaining. 00000020:00200000:1.0:1787826300.395637:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 484 items remaining. 00000020:00200000:1.0:1787826300.395638:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 483 items remaining. 00000020:00200000:1.0:1787826300.395640:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 482 items remaining. 00000020:00200000:1.0:1787826300.395641:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 481 items remaining. 00000020:00200000:1.0:1787826300.395643:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 480 items remaining. 00000020:00200000:1.0:1787826300.395644:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 479 items remaining. 00000020:00200000:1.0:1787826300.395646:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 478 items remaining. 00000020:00200000:1.0:1787826300.395648:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 477 items remaining. 00000020:00200000:1.0:1787826300.395650:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 476 items remaining. 00000020:00200000:1.0:1787826300.395653:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 475 items remaining. 00000020:00200000:1.0:1787826300.395654:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 474 items remaining. 00000020:00200000:1.0:1787826300.395656:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 473 items remaining. 00000020:00200000:1.0:1787826300.395658:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 472 items remaining. 00000020:00200000:1.0:1787826300.395659:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 471 items remaining. 00000020:00200000:1.0:1787826300.395661:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 470 items remaining. 00000020:00200000:1.0:1787826300.395663:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 469 items remaining. 00000020:00200000:1.0:1787826300.395664:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 468 items remaining. 00000020:00200000:1.0:1787826300.395666:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 467 items remaining. 00000020:00200000:1.0:1787826300.395667:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 466 items remaining. 00000020:00200000:1.0:1787826300.395669:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 465 items remaining. 00000020:00200000:1.0:1787826300.395671:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 464 items remaining. 00000020:00200000:1.0:1787826300.395673:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 463 items remaining. 00000020:00200000:1.0:1787826300.395674:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 462 items remaining. 00000020:00200000:1.0:1787826300.395676:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 461 items remaining. 00000020:00200000:1.0:1787826300.395677:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 460 items remaining. 00000020:00200000:1.0:1787826300.395679:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 459 items remaining. 00000020:00200000:1.0:1787826300.395681:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 458 items remaining. 00000020:00200000:1.0:1787826300.395683:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 457 items remaining. 00000020:00200000:1.0:1787826300.395684:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 456 items remaining. 00000020:00200000:1.0:1787826300.395686:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 455 items remaining. 00000020:00200000:1.0:1787826300.395688:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 454 items remaining. 00000020:00200000:1.0:1787826300.395690:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 453 items remaining. 00000020:00200000:1.0:1787826300.395691:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 452 items remaining. 00000020:00200000:1.0:1787826300.395693:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 451 items remaining. 00000020:00200000:1.0:1787826300.395695:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 450 items remaining. 00000020:00200000:1.0:1787826300.395697:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 449 items remaining. 00000020:00200000:1.0:1787826300.395698:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 448 items remaining. 00000020:00200000:1.0:1787826300.395700:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 447 items remaining. 00000020:00200000:1.0:1787826300.395702:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 446 items remaining. 00000020:00200000:1.0:1787826300.395705:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 445 items remaining. 00000020:00200000:1.0:1787826300.395707:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 444 items remaining. 00000020:00200000:1.0:1787826300.395709:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 443 items remaining. 00000020:00200000:1.0:1787826300.395711:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 442 items remaining. 00000020:00200000:1.0:1787826300.395712:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 441 items remaining. 00000020:00200000:1.0:1787826300.395714:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 440 items remaining. 00000020:00200000:1.0:1787826300.395715:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 439 items remaining. 00000020:00200000:1.0:1787826300.395717:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 438 items remaining. 00000020:00200000:1.0:1787826300.395719:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 437 items remaining. 00000020:00200000:1.0:1787826300.395720:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 436 items remaining. 00000020:00200000:1.0:1787826300.395722:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 435 items remaining. 00000020:00200000:1.0:1787826300.395723:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 434 items remaining. 00000020:00200000:1.0:1787826300.395725:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 433 items remaining. 00000020:00200000:1.0:1787826300.395726:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 432 items remaining. 00000020:00200000:1.0:1787826300.395728:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 431 items remaining. 00000020:00200000:1.0:1787826300.395729:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 430 items remaining. 00000020:00200000:1.0:1787826300.395732:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 429 items remaining. 00000020:00200000:1.0:1787826300.395734:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 428 items remaining. 00000020:00200000:1.0:1787826300.395736:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 427 items remaining. 00000020:00200000:1.0:1787826300.395738:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 426 items remaining. 00000020:00200000:1.0:1787826300.395739:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 425 items remaining. 00000020:00200000:1.0:1787826300.395741:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 424 items remaining. 00000020:00200000:1.0:1787826300.395743:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 423 items remaining. 00000020:00200000:1.0:1787826300.395745:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 422 items remaining. 00000020:00200000:1.0:1787826300.395746:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 421 items remaining. 00000020:00200000:1.0:1787826300.396009:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 420 items remaining. 00000020:00200000:1.0:1787826300.396012:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 419 items remaining. 00000020:00200000:1.0:1787826300.396014:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 418 items remaining. 00000020:00200000:1.0:1787826300.396015:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 417 items remaining. 00000020:00200000:1.0:1787826300.396017:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 416 items remaining. 00000020:00200000:1.0:1787826300.396019:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 415 items remaining. 00000020:00200000:1.0:1787826300.396020:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 414 items remaining. 00000020:00200000:1.0:1787826300.396023:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 413 items remaining. 00000020:00200000:1.0:1787826300.396025:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 412 items remaining. 00000020:00200000:1.0:1787826300.396027:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 411 items remaining. 00000020:00200000:1.0:1787826300.396028:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 410 items remaining. 00000020:00200000:1.0:1787826300.396030:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 409 items remaining. 00000020:00200000:1.0:1787826300.396032:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 408 items remaining. 00000020:00200000:1.0:1787826300.396034:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 407 items remaining. 00000020:00200000:1.0:1787826300.396035:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 406 items remaining. 00000020:00200000:1.0:1787826300.396037:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 405 items remaining. 00000020:00200000:1.0:1787826300.396039:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 404 items remaining. 00000020:00200000:1.0:1787826300.396041:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 403 items remaining. 00000020:00200000:1.0:1787826300.396042:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 402 items remaining. 00000020:00200000:1.0:1787826300.396044:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 401 items remaining. 00000020:00200000:1.0:1787826300.396088:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 400 items remaining. 00000020:00200000:1.0:1787826300.396091:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 399 items remaining. 00000020:00200000:1.0:1787826300.396093:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 398 items remaining. 00000020:00200000:1.0:1787826300.396096:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 397 items remaining. 00000020:00200000:1.0:1787826300.396097:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 396 items remaining. 00000020:00200000:1.0:1787826300.396099:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 395 items remaining. 00000020:00200000:1.0:1787826300.396101:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 394 items remaining. 00000020:00200000:1.0:1787826300.396103:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 393 items remaining. 00000020:00200000:1.0:1787826300.396104:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 392 items remaining. 00000020:00200000:1.0:1787826300.396106:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 391 items remaining. 00000020:00200000:1.0:1787826300.396108:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 390 items remaining. 00000020:00200000:1.0:1787826300.396109:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 389 items remaining. 00000020:00200000:1.0:1787826300.396111:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 388 items remaining. 00000020:00200000:1.0:1787826300.396113:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 387 items remaining. 00000020:00200000:1.0:1787826300.396114:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 386 items remaining. 00000020:00200000:1.0:1787826300.396116:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 385 items remaining. 00000020:00200000:1.0:1787826300.396118:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 384 items remaining. 00000020:00200000:1.0:1787826300.396119:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 383 items remaining. 00000020:00200000:1.0:1787826300.396121:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 382 items remaining. 00000020:00200000:1.0:1787826300.396123:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 381 items remaining. 00000020:00200000:1.0:1787826300.396125:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 380 items remaining. 00000020:00200000:1.0:1787826300.396127:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 379 items remaining. 00000020:00200000:1.0:1787826300.396129:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 378 items remaining. 00000020:00200000:1.0:1787826300.396131:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 377 items remaining. 00000020:00200000:1.0:1787826300.396132:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 376 items remaining. 00000020:00200000:1.0:1787826300.396134:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 375 items remaining. 00000020:00200000:1.0:1787826300.396197:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 374 items remaining. 00000020:00200000:1.0:1787826300.396200:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 373 items remaining. 00000020:00200000:1.0:1787826300.396202:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 372 items remaining. 00000020:00200000:1.0:1787826300.396204:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 371 items remaining. 00000020:00200000:1.0:1787826300.396206:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 370 items remaining. 00000020:00200000:1.0:1787826300.396208:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 369 items remaining. 00000020:00200000:1.0:1787826300.396209:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 368 items remaining. 00000020:00200000:1.0:1787826300.396211:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 367 items remaining. 00000020:00200000:1.0:1787826300.396213:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 366 items remaining. 00000020:00200000:1.0:1787826300.396215:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 365 items remaining. 00000020:00200000:1.0:1787826300.396217:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 364 items remaining. 00000020:00200000:1.0:1787826300.396219:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 363 items remaining. 00000020:00200000:1.0:1787826300.396221:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 362 items remaining. 00000020:00200000:1.0:1787826300.396223:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 361 items remaining. 00000020:00200000:1.0:1787826300.396225:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 360 items remaining. 00000020:00200000:1.0:1787826300.396227:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 359 items remaining. 00000020:00200000:1.0:1787826300.396229:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 358 items remaining. 00000020:00200000:1.0:1787826300.396231:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 357 items remaining. 00000020:00200000:1.0:1787826300.396233:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 356 items remaining. 00000020:00200000:1.0:1787826300.396234:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 355 items remaining. 00000020:00200000:1.0:1787826300.396236:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 354 items remaining. 00000020:00200000:1.0:1787826300.396237:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 353 items remaining. 00000020:00200000:1.0:1787826300.396239:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 352 items remaining. 00000020:00200000:1.0:1787826300.396241:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 351 items remaining. 00000020:00200000:1.0:1787826300.396242:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 350 items remaining. 00000020:00200000:1.0:1787826300.396245:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 349 items remaining. 00000020:00200000:1.0:1787826300.396247:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 348 items remaining. 00000020:00200000:1.0:1787826300.396249:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 347 items remaining. 00000020:00200000:1.0:1787826300.396251:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 346 items remaining. 00000020:00200000:1.0:1787826300.396253:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 345 items remaining. 00000020:00200000:1.0:1787826300.396255:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 344 items remaining. 00000020:00200000:1.0:1787826300.396257:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 343 items remaining. 00000020:00200000:1.0:1787826300.396258:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 342 items remaining. 00000020:00200000:1.0:1787826300.396260:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 341 items remaining. 00000020:00200000:1.0:1787826300.396262:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 340 items remaining. 00000020:00200000:1.0:1787826300.396264:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 339 items remaining. 00000020:00200000:1.0:1787826300.396266:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 338 items remaining. 00000020:00200000:1.0:1787826300.396268:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 337 items remaining. 00000020:00200000:1.0:1787826300.396270:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 336 items remaining. 00000020:00200000:1.0:1787826300.396272:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 335 items remaining. 00000020:00200000:1.0:1787826300.396274:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 334 items remaining. 00000020:00200000:1.0:1787826300.396275:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 333 items remaining. 00000020:00200000:1.0:1787826300.396277:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 332 items remaining. 00000020:00200000:1.0:1787826300.396279:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 331 items remaining. 00000020:00200000:1.0:1787826300.396290:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 330 items remaining. 00000020:00200000:1.0:1787826300.396292:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 329 items remaining. 00000020:00200000:1.0:1787826300.396294:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 328 items remaining. 00000020:00200000:1.0:1787826300.396296:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 327 items remaining. 00000020:00200000:1.0:1787826300.396298:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 326 items remaining. 00000020:00200000:1.0:1787826300.396300:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 325 items remaining. 00000020:00200000:1.0:1787826300.396302:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 324 items remaining. 00000020:00200000:1.0:1787826300.396303:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 323 items remaining. 00000020:00200000:1.0:1787826300.396305:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 322 items remaining. 00000020:00200000:1.0:1787826300.396307:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 321 items remaining. 00000020:00200000:1.0:1787826300.396308:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 320 items remaining. 00000020:00200000:1.0:1787826300.396310:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 319 items remaining. 00000020:00200000:1.0:1787826300.396311:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 318 items remaining. 00000020:00200000:1.0:1787826300.396314:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 317 items remaining. 00000020:00200000:1.0:1787826300.396316:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 316 items remaining. 00000020:00200000:1.0:1787826300.396318:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 315 items remaining. 00000020:00200000:1.0:1787826300.396320:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 314 items remaining. 00000020:00200000:1.0:1787826300.396321:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 313 items remaining. 00000020:00200000:1.0:1787826300.396323:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 312 items remaining. 00000020:00200000:1.0:1787826300.396325:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 311 items remaining. 00000020:00200000:1.0:1787826300.396326:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 310 items remaining. 00000020:00200000:1.0:1787826300.396328:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 309 items remaining. 00000020:00200000:1.0:1787826300.396330:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 308 items remaining. 00000020:00200000:1.0:1787826300.396331:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 307 items remaining. 00000020:00200000:1.0:1787826300.396333:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 306 items remaining. 00000020:00200000:1.0:1787826300.396335:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 305 items remaining. 00000020:00200000:1.0:1787826300.396336:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 304 items remaining. 00000020:00200000:1.0:1787826300.396338:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 303 items remaining. 00000020:00200000:1.0:1787826300.396340:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 302 items remaining. 00000020:00200000:1.0:1787826300.396342:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 301 items remaining. 00000020:00200000:1.0:1787826300.396344:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 300 items remaining. 00000020:00200000:1.0:1787826300.396346:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 299 items remaining. 00000020:00200000:1.0:1787826300.396348:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 298 items remaining. 00000020:00200000:1.0:1787826300.396349:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 297 items remaining. 00000020:00200000:1.0:1787826300.396351:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 296 items remaining. 00000020:00200000:1.0:1787826300.396353:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 295 items remaining. 00000020:00200000:1.0:1787826300.396355:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 294 items remaining. 00000020:00200000:1.0:1787826300.396356:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 293 items remaining. 00000020:00200000:1.0:1787826300.396358:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 292 items remaining. 00000020:00200000:1.0:1787826300.396359:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 291 items remaining. 00000020:00200000:1.0:1787826300.396361:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 290 items remaining. 00000020:00200000:1.0:1787826300.396362:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 289 items remaining. 00000020:00200000:1.0:1787826300.396364:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 288 items remaining. 00000020:00200000:1.0:1787826300.396365:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 287 items remaining. 00000020:00200000:1.0:1787826300.396367:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 286 items remaining. 00000020:00200000:1.0:1787826300.396369:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 285 items remaining. 00000020:00200000:1.0:1787826300.396371:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 284 items remaining. 00000020:00200000:1.0:1787826300.396373:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 283 items remaining. 00000020:00200000:1.0:1787826300.396375:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 282 items remaining. 00000020:00200000:1.0:1787826300.396377:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 281 items remaining. 00000020:00200000:1.0:1787826300.396379:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 280 items remaining. 00000020:00200000:1.0:1787826300.396381:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 279 items remaining. 00000020:00200000:1.0:1787826300.396383:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 278 items remaining. 00000020:00200000:1.0:1787826300.396384:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 277 items remaining. 00000020:00200000:1.0:1787826300.396386:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 276 items remaining. 00000020:00200000:1.0:1787826300.396388:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 275 items remaining. 00000020:00200000:1.0:1787826300.396390:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 274 items remaining. 00000020:00200000:1.0:1787826300.396391:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 273 items remaining. 00000020:00200000:1.0:1787826300.396393:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 272 items remaining. 00000020:00200000:1.0:1787826300.396395:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 271 items remaining. 00000020:00200000:1.0:1787826300.396396:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 270 items remaining. 00000020:00200000:1.0:1787826300.396398:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 269 items remaining. 00000020:00200000:1.0:1787826300.396399:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 268 items remaining. 00000020:00200000:1.0:1787826300.396401:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 267 items remaining. 00000020:00200000:1.0:1787826300.396402:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 266 items remaining. 00000020:00200000:1.0:1787826300.396403:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 265 items remaining. 00000020:00200000:1.0:1787826300.396405:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 264 items remaining. 00000020:00200000:1.0:1787826300.396407:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 263 items remaining. 00000020:00200000:1.0:1787826300.396408:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 262 items remaining. 00000020:00200000:1.0:1787826300.396410:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 261 items remaining. 00000020:00200000:1.0:1787826300.396924:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 260 items remaining. 00000020:00200000:1.0:1787826300.396931:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 259 items remaining. 00000020:00200000:1.0:1787826300.396933:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 258 items remaining. 00000020:00200000:1.0:1787826300.396935:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 257 items remaining. 00000020:00200000:1.0:1787826300.396936:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000020:00200000:1.0:1787826300.396938:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:1.0:1787826300.396940:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:1.0:1787826300.396942:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:1.0:1787826300.396944:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:1.0:1787826300.396946:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:1.0:1787826300.396948:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:1.0:1787826300.396949:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:1.0:1787826300.396951:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:1.0:1787826300.396953:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:1.0:1787826300.396955:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:1.0:1787826300.396957:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:1.0:1787826300.396959:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:1.0:1787826300.396961:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:1.0:1787826300.396963:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:1.0:1787826300.396965:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:1.0:1787826300.396967:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:1.0:1787826300.396969:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:1.0:1787826300.396971:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:1.0:1787826300.396973:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:1.0:1787826300.396975:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:1.0:1787826300.396977:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:1.0:1787826300.396979:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:1.0:1787826300.396981:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:1.0:1787826300.396982:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:1.0:1787826300.396985:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:1.0:1787826300.396987:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:1.0:1787826300.396989:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:1.0:1787826300.396990:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:1.0:1787826300.396992:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:1.0:1787826300.396994:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:1.0:1787826300.396995:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:1.0:1787826300.396997:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:1.0:1787826300.396999:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:1.0:1787826300.397001:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:1.0:1787826300.397003:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:1.0:1787826300.397005:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:1.0:1787826300.397007:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:1.0:1787826300.397009:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:1.0:1787826300.397011:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:1.0:1787826300.397012:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:1.0:1787826300.397014:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:1.0:1787826300.397016:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:1.0:1787826300.397018:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:1.0:1787826300.397020:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:1.0:1787826300.397022:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:1.0:1787826300.397024:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:1.0:1787826300.397026:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:1.0:1787826300.397028:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:1.0:1787826300.397030:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:1.0:1787826300.397032:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:1.0:1787826300.397034:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:1.0:1787826300.397036:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:1.0:1787826300.397038:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:1.0:1787826300.397040:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:1.0:1787826300.397042:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:1.0:1787826300.397044:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:1.0:1787826300.397106:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:1.0:1787826300.397109:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:1.0:1787826300.397111:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:1.0:1787826300.397113:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:1.0:1787826300.397115:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:1.0:1787826300.397117:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:1.0:1787826300.397118:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:1.0:1787826300.397120:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:1.0:1787826300.397122:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:1.0:1787826300.397123:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:1.0:1787826300.397126:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:1.0:1787826300.397127:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:1.0:1787826300.397129:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:1.0:1787826300.397131:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:1.0:1787826300.397132:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:1.0:1787826300.397134:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:1.0:1787826300.397135:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:1.0:1787826300.397188:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:1.0:1787826300.397191:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:1.0:1787826300.397193:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:1.0:1787826300.397196:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:1.0:1787826300.397198:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:1.0:1787826300.397200:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:1.0:1787826300.397201:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:1.0:1787826300.397204:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:1.0:1787826300.397206:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:1.0:1787826300.397207:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:1.0:1787826300.397210:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:1.0:1787826300.397212:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:1.0:1787826300.397214:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:1.0:1787826300.397216:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:1.0:1787826300.397218:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:1.0:1787826300.397219:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:1.0:1787826300.397222:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:1.0:1787826300.397223:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:1.0:1787826300.397226:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:1.0:1787826300.397227:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:1.0:1787826300.397229:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:1.0:1787826300.397231:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:1.0:1787826300.397232:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:1.0:1787826300.397234:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:1.0:1787826300.397235:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:1.0:1787826300.397237:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:1.0:1787826300.397239:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:1.0:1787826300.397242:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:1.0:1787826300.397244:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:1.0:1787826300.397245:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:1.0:1787826300.397247:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:1.0:1787826300.397249:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:1.0:1787826300.397251:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:1.0:1787826300.397253:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:1.0:1787826300.397255:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:1.0:1787826300.397256:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:1.0:1787826300.397258:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:1.0:1787826300.397260:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:1.0:1787826300.397263:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:1.0:1787826300.397265:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:1.0:1787826300.397267:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:1.0:1787826300.397269:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:1.0:1787826300.397271:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:1.0:1787826300.397272:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:1.0:1787826300.397275:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:1.0:1787826300.397277:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:1.0:1787826300.397279:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:1.0:1787826300.397691:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:1.0:1787826300.397694:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:1.0:1787826300.397696:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:1.0:1787826300.397698:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:1.0:1787826300.397700:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:1.0:1787826300.397702:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:1.0:1787826300.397704:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:1.0:1787826300.397706:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:1.0:1787826300.397708:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:1.0:1787826300.397710:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:1.0:1787826300.397712:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:1.0:1787826300.397714:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:1.0:1787826300.397716:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:1.0:1787826300.397718:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:1.0:1787826300.397720:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:1.0:1787826300.397722:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:1.0:1787826300.397723:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:1.0:1787826300.397725:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:1.0:1787826300.397727:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:1.0:1787826300.397728:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:1.0:1787826300.397730:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:1.0:1787826300.397731:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:1.0:1787826300.397733:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:1.0:1787826300.397735:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:1.0:1787826300.397737:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:1.0:1787826300.397739:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:1.0:1787826300.397741:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:1.0:1787826300.397742:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:1.0:1787826300.397744:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:1.0:1787826300.397746:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:1.0:1787826300.398207:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:1.0:1787826300.398210:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:1.0:1787826300.398212:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:1.0:1787826300.398214:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:1.0:1787826300.398216:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:1.0:1787826300.398219:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:1.0:1787826300.398221:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:1.0:1787826300.398223:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:1.0:1787826300.398225:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:1.0:1787826300.398227:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:1.0:1787826300.398228:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:1.0:1787826300.398230:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:1.0:1787826300.398233:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:1.0:1787826300.398235:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:1.0:1787826300.398237:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:1.0:1787826300.398239:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:1.0:1787826300.398241:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:1.0:1787826300.398243:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:1.0:1787826300.398245:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:1.0:1787826300.398247:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:1.0:1787826300.398249:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:1.0:1787826300.398251:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:1.0:1787826300.398253:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:1.0:1787826300.398256:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:1.0:1787826300.398258:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:1.0:1787826300.398260:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:1.0:1787826300.398262:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:1.0:1787826300.398264:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:1.0:1787826300.398266:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:1.0:1787826300.398268:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:1.0:1787826300.398270:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:1.0:1787826300.398272:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:1.0:1787826300.398274:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:1.0:1787826300.398276:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:1.0:1787826300.398277:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:1.0:1787826300.398288:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:1.0:1787826300.398290:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:1.0:1787826300.398292:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:1.0:1787826300.398294:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:1.0:1787826300.398297:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:1.0:1787826300.398299:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:1.0:1787826300.398301:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:1.0:1787826300.398303:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:1.0:1787826300.398305:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:1.0:1787826300.398308:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:1.0:1787826300.398310:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:1.0:1787826300.398312:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:1.0:1787826300.398313:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:1.0:1787826300.398315:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:1.0:1787826300.398317:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:1.0:1787826300.398319:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:1.0:1787826300.398320:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:1.0:1787826300.398322:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:1.0:1787826300.398324:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:1.0:1787826300.398326:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:1.0:1787826300.398328:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:1.0:1787826300.398330:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:1.0:1787826300.398331:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:1.0:1787826300.398333:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:1.0:1787826300.398335:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:1.0:1787826300.398336:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:1.0:1787826300.398338:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:1.0:1787826300.398340:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:1.0:1787826300.398342:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:1.0:1787826300.398344:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:1.0:1787826300.398346:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:1.0:1787826300.398347:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:1.0:1787826300.398349:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:1.0:1787826300.398350:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:1.0:1787826300.398352:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:1.0:1787826300.398354:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:1.0:1787826300.398356:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:1.0:1787826300.398358:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:1.0:1787826300.398359:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:1.0:1787826300.398361:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:1.0:1787826300.398363:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:1.0:1787826300.398366:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:1.0:1787826300.398368:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:1.0:1787826300.398370:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:1.0:1787826300.398371:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:1.0:1787826300.398373:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:1.0:1787826300.398374:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:1.0:1787826300.398376:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:1.0:1787826300.398377:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:1.0:1787826300.398379:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:1.0:1787826300.398380:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:1.0:1787826300.398382:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:1.0:1787826300.398383:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:1.0:1787826300.398385:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:1.0:1787826300.398386:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:1.0:1787826300.398387:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:1.0:1787826300.398389:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:1.0:1787826300.398390:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:1.0:1787826300.398391:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:1.0:1787826300.398393:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:1.0:1787826300.398394:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:1.0:1787826300.398396:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:1.0:1787826300.398397:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:1.0:1787826300.398399:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:1.0:1787826300.398400:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:1.0:1787826300.398402:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:1.0:1787826300.398403:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:1.0:1787826300.398405:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:1.0:1787826300.398407:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:1.0:1787826300.398408:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:1.0:1787826300.398410:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1787826300.399107:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1787826300.399301:0:2348:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907da70a3c00 00000100:00100000:1.0:1787826300.399346:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011431424:192.168.203.104@tcp:3:lfs.0 00000080:00200000:2.0:1787826300.399585:0:67244:0:(file.c:2207:ll_file_io_generic()) Range unlock [0, 511] 00000080:00200000:2.0:1787826300.399603:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826300.399643:0:67244:0:(file.c:2240:ll_file_io_generic()) f204c.sanity-flr: 1 io complete with rc: 0, result: 2097152, restart: 0 00000020:00200000:2.0:1787826300.399647:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826300.399743:0:67244:0:(file.c:2326:ll_file_io_generic()) iot: 1, result: 2097152 00000080:00200080:1.0:1787826300.399843:0:67244:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908) cmd=400866b1 arg=0 00000080:00200080:1.0:1787826300.399858:0:67244:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908) cmd=400866b1 arg=2 00000080:00200008:1.0:1787826300.399867:0:67244:0:(file.c:6313:ll_getattr_dentry()) START file f204c.sanity-flr:[0x200000404:0x203:0x0](ffff907d93310908), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826300.399877:0:67244:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908),name=f204c.sanity-flr 00000002:00010000:1.0:1787826300.399905:0:67244:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x203:0x0]) in obj [0x200000404:0x203:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826300.399929:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826300.399946:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1787826300.399955:0:67244:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x203:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826300.399964:0:67244:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) 00000080:00010000:1.0:1787826300.399968:0:67244:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) for lock 0xe5f736940357b8ea 00000080:00010000:1.0:1787826300.399978:0:67244:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b8ea from it ffff9dd141f63cd8 00010000:00010000:1.0:1787826300.399982:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826300.399992:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1787826300.400022:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/1, layout version 8 restore needed 0 00020000:00200000:1.0:1787826300.400038:0:67244:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826300.400043:0:67244:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826300.400047:0:67244:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x203:0x0] 00000008:00010000:1.0:1787826300.400072:0:67244:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826300.400077:0:67244:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c32fb0, osc lock ffff907d962f5998, flags 0x400000400001002 00010000:00010000:1.0:1787826300.400098:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826300.400111:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1787826300.400123:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826300.400126:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826300.400208:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00200000:1.0:1787826300.400223:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826300.400234:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:1.0:1787826300.400239:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5791148 (ffff907d884a7ea8/0) at cl_lock_release():245 00000020:00010000:1.0:1787826300.400242:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5791148 (ffff907d884a7ea8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826300.400245:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c32fb0 (ffff907d884a7b48/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826300.400250:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5791148 (ffff907d884a7ea8/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826300.400253:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c32fb0 (ffff907d884a7ea8/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826300.400264:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/1, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826300.400300:0:67244:0:(file.c:2652:do_file_write_iter()) START file f204c.sanity-flr:[0x200000404:0x203:0x0], ppos: 0, count: 1048576 00000080:00200000:1.0:1787826300.400308:0:67244:0:(file.c:2090:ll_file_io_generic()) f204c.sanity-flr: write ppos: 0, bytes: 1048576 00000080:00200000:1.0:1787826300.400317:0:67244:0:(file.c:1848:ll_io_set_mirror()) f204c.sanity-flr: desiginated mirror: 2 00000020:00200000:1.0:1787826300.400324:0:67244:0:(cl_io.c:218:cl_io_rw_init()) header@000000008ae07a1f[0x110, 2, [0x200000404:0x203:0x0] hash] 00000020:00200000:1.0:1787826300.400327:0:67244:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [0, 1048576) 2048 0 00000080:00200000:1.0:1787826300.400330:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 8 restore needed 0 00000080:00200000:1.0:1787826300.400344:0:67244:0:(file.c:2170:ll_file_io_generic()) Range lock [0, 255] 00020000:00200000:1.0:1787826300.400353:0:67244:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:1.0:1787826300.400357:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x11 00000080:00200000:1.0:1787826300.400374:0:67244:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 255] 00000008:00010000:1.0:1787826300.400383:0:67244:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 11 00000008:00010000:1.0:1787826300.400387:0:67244:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c32fb0, osc lock ffff907d962f5998, flags 0x40000 00000080:00200000:1.0:1787826300.400397:0:67244:0:(vvp_io.c:1339:vvp_io_write_start()) f204c.sanity-flr: write [0, 1048576) append 0 append lockpos 0 00000080:00200000:1.0:1787826300.400432:0:67244:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908), size=1048576 (max 335544320), offset=0=0x0, pages 256 (max 81920), lockless, parallel 00000020:00200000:1.0:1787826300.400907:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:1.0:1787826300.400911:0:67244:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 1048576 00000080:00200000:1.0:1787826300.400923:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:1.0:1787826300.400928:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98af23f8 (ffff907d884a7ea8/0) at cl_lock_release():245 00000020:00010000:1.0:1787826300.400931:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98af23f8 (ffff907d884a7ea8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826300.400934:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c32fb0 (ffff907d884a7b48/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826300.400937:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98af23f8 (ffff907d884a7ea8/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826300.400939:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c32fb0 (ffff907d884a7ea8/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826300.400946:0:67244:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:1.0:1787826300.400954:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1787826300.400980:0:58785:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907d908aa900 00000008:00100000:1.0:1787826300.402015:0:58785:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d908a2580 - object 0x280000400:779 offset 0<>1048576 00000100:00100000:0.0:1787826300.402087:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1787826300.402248:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011431552:192.168.203.104@tcp:4:lfs.0 00000100:00100000:0.0:1787826300.434018:0:2348:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0001-osc-ffff907d986d0000: committing for last_committed 17179873840 gen 1 00000008:00010000:0.0:1787826300.434166:0:2348:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907d908aa900 00000020:00200000:0.0:1787826300.434175:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:0.0:1787826300.434179:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:0.0:1787826300.434181:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:0.0:1787826300.434182:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:0.0:1787826300.434184:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:0.0:1787826300.434186:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:0.0:1787826300.434188:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:0.0:1787826300.434190:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:0.0:1787826300.434192:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:0.0:1787826300.434194:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:0.0:1787826300.434195:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:0.0:1787826300.434197:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:0.0:1787826300.434198:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:0.0:1787826300.434200:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:0.0:1787826300.434201:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:0.0:1787826300.434203:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:0.0:1787826300.434206:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:0.0:1787826300.434208:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:0.0:1787826300.434210:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:0.0:1787826300.434212:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:0.0:1787826300.434213:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:0.0:1787826300.434215:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:0.0:1787826300.434217:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:0.0:1787826300.434218:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:0.0:1787826300.434220:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:0.0:1787826300.434221:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:0.0:1787826300.434223:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:0.0:1787826300.434225:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:0.0:1787826300.434227:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:0.0:1787826300.434228:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:0.0:1787826300.434230:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:0.0:1787826300.434232:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:0.0:1787826300.434234:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:0.0:1787826300.434236:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:0.0:1787826300.434238:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:0.0:1787826300.434239:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:0.0:1787826300.434241:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:0.0:1787826300.434242:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:0.0:1787826300.434244:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:0.0:1787826300.434246:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:0.0:1787826300.434248:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:0.0:1787826300.434249:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:0.0:1787826300.434251:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:0.0:1787826300.434253:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:0.0:1787826300.434254:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:0.0:1787826300.434256:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:0.0:1787826300.434258:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:0.0:1787826300.434259:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:0.0:1787826300.434261:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:0.0:1787826300.434263:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:0.0:1787826300.434265:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:0.0:1787826300.434267:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:0.0:1787826300.434269:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:0.0:1787826300.434271:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:0.0:1787826300.434273:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:0.0:1787826300.434274:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:0.0:1787826300.434276:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:0.0:1787826300.434278:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:0.0:1787826300.434291:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:0.0:1787826300.434294:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:0.0:1787826300.434296:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:0.0:1787826300.434297:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:0.0:1787826300.434299:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:0.0:1787826300.434300:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:0.0:1787826300.434302:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:0.0:1787826300.434303:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:0.0:1787826300.434305:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:0.0:1787826300.434306:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:0.0:1787826300.434308:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:0.0:1787826300.434309:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:0.0:1787826300.434311:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:0.0:1787826300.434313:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:0.0:1787826300.434315:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:0.0:1787826300.434317:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:0.0:1787826300.434318:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:0.0:1787826300.434320:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:0.0:1787826300.434322:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:0.0:1787826300.434323:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:0.0:1787826300.434325:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:0.0:1787826300.434326:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:0.0:1787826300.434329:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:0.0:1787826300.434331:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:0.0:1787826300.434332:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:0.0:1787826300.434334:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:0.0:1787826300.434336:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:0.0:1787826300.434337:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:0.0:1787826300.434339:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:0.0:1787826300.434340:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:0.0:1787826300.434342:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:0.0:1787826300.434344:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:0.0:1787826300.434346:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:0.0:1787826300.434348:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:0.0:1787826300.434350:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:0.0:1787826300.434352:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:0.0:1787826300.434354:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:0.0:1787826300.434355:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:0.0:1787826300.434357:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:0.0:1787826300.434359:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:0.0:1787826300.434361:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:0.0:1787826300.434363:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:0.0:1787826300.434365:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:0.0:1787826300.434367:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:0.0:1787826300.434369:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:0.0:1787826300.434371:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:0.0:1787826300.434373:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:0.0:1787826300.434375:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:0.0:1787826300.434377:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:0.0:1787826300.434379:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:0.0:1787826300.434381:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:0.0:1787826300.434383:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:0.0:1787826300.434384:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:0.0:1787826300.434386:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:0.0:1787826300.434388:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:0.0:1787826300.434390:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:0.0:1787826300.434392:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:0.0:1787826300.434393:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:0.0:1787826300.434395:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:0.0:1787826300.434397:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:0.0:1787826300.434398:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:0.0:1787826300.434400:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:0.0:1787826300.434401:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:0.0:1787826300.434403:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:0.0:1787826300.434405:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:0.0:1787826300.434407:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:0.0:1787826300.434408:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:0.0:1787826300.434410:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:0.0:1787826300.434412:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:0.0:1787826300.434414:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:0.0:1787826300.434417:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:0.0:1787826300.434418:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:0.0:1787826300.434420:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:0.0:1787826300.434421:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:0.0:1787826300.434422:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:0.0:1787826300.434424:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:0.0:1787826300.434426:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:0.0:1787826300.434428:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:0.0:1787826300.434430:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:0.0:1787826300.434432:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:0.0:1787826300.434434:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:0.0:1787826300.434435:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:0.0:1787826300.434436:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:0.0:1787826300.434438:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:0.0:1787826300.434439:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:0.0:1787826300.434441:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:0.0:1787826300.434443:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:0.0:1787826300.434444:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:0.0:1787826300.434446:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:0.0:1787826300.434448:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:0.0:1787826300.434450:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:0.0:1787826300.434452:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:0.0:1787826300.434453:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:0.0:1787826300.434455:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:0.0:1787826300.434456:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:0.0:1787826300.434458:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:0.0:1787826300.434460:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:0.0:1787826300.434461:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:0.0:1787826300.434463:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:0.0:1787826300.434465:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:0.0:1787826300.434467:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:0.0:1787826300.434469:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:0.0:1787826300.434472:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:0.0:1787826300.434473:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:0.0:1787826300.434475:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:0.0:1787826300.434477:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:0.0:1787826300.434479:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:0.0:1787826300.434480:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:0.0:1787826300.434482:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:0.0:1787826300.434484:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:0.0:1787826300.434485:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:0.0:1787826300.434487:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:0.0:1787826300.434489:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:0.0:1787826300.434490:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:0.0:1787826300.434491:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:0.0:1787826300.434493:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:0.0:1787826300.434494:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:0.0:1787826300.434496:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:0.0:1787826300.434499:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:0.0:1787826300.434500:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:0.0:1787826300.434502:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:0.0:1787826300.434504:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:0.0:1787826300.434505:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:0.0:1787826300.434507:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:0.0:1787826300.434509:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:0.0:1787826300.434511:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:0.0:1787826300.434513:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:0.0:1787826300.434514:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:0.0:1787826300.434516:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:0.0:1787826300.434518:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:0.0:1787826300.434520:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:0.0:1787826300.434522:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:0.0:1787826300.434524:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:0.0:1787826300.434526:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:0.0:1787826300.434527:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:0.0:1787826300.434530:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:0.0:1787826300.434532:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:0.0:1787826300.434533:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:0.0:1787826300.434535:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:0.0:1787826300.434536:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:0.0:1787826300.434538:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:0.0:1787826300.434540:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:0.0:1787826300.434542:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:0.0:1787826300.434544:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:0.0:1787826300.434546:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:0.0:1787826300.434548:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:0.0:1787826300.434550:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:0.0:1787826300.434551:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:0.0:1787826300.434553:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:0.0:1787826300.434555:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:0.0:1787826300.434557:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:0.0:1787826300.434560:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:0.0:1787826300.434562:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:0.0:1787826300.434563:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:0.0:1787826300.434565:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:0.0:1787826300.434567:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:0.0:1787826300.434568:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:0.0:1787826300.434569:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:0.0:1787826300.434571:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:0.0:1787826300.434573:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:0.0:1787826300.434574:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:0.0:1787826300.434576:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:0.0:1787826300.434578:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:0.0:1787826300.434580:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:0.0:1787826300.434582:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:0.0:1787826300.434584:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:0.0:1787826300.434586:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:0.0:1787826300.434588:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:0.0:1787826300.434590:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:0.0:1787826300.434592:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:0.0:1787826300.434594:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:0.0:1787826300.434596:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:0.0:1787826300.434597:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:0.0:1787826300.434599:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:0.0:1787826300.434601:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:0.0:1787826300.434602:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:0.0:1787826300.434604:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:0.0:1787826300.434639:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:0.0:1787826300.434717:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:0.0:1787826300.434721:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:0.0:1787826300.434722:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:0.0:1787826300.434786:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:0.0:1787826300.434789:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:0.0:1787826300.434791:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:0.0:1787826300.434793:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:0.0:1787826300.434795:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:0.0:1787826300.434796:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:0.0:1787826300.434798:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:0.0:1787826300.434800:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:0.0:1787826300.434802:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:0.0:1787826300.434804:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:0.0:1787826300.434806:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:0.0:1787826300.434808:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:0.0:1787826300.434810:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:0.0:1787826300.434812:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:0.0:1787826300.434814:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:0.0:1787826300.434815:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:0.0:1787826300.434817:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1787826300.435057:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1787826300.435233:0:2348:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907d908aa900 00000100:00100000:0.0:1787826300.435270:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011431552:192.168.203.104@tcp:4:lfs.0 00000080:00200000:1.0:1787826300.435395:0:67244:0:(file.c:2207:ll_file_io_generic()) Range unlock [0, 255] 00000080:00200000:1.0:1787826300.435412:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826300.435419:0:67244:0:(file.c:2240:ll_file_io_generic()) f204c.sanity-flr: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:1.0:1787826300.435434:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000020:00200000:1.0:1787826300.435437:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826300.435448:0:67244:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200080:1.0:1787826300.435474:0:67244:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908) cmd=400866b1 arg=0 00000080:00200080:1.0:1787826300.435488:0:67244:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908) cmd=400866b1 arg=2 00000080:00200008:1.0:1787826300.435496:0:67244:0:(file.c:6313:ll_getattr_dentry()) START file f204c.sanity-flr:[0x200000404:0x203:0x0](ffff907d93310908), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826300.435506:0:67244:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908),name=f204c.sanity-flr 00000002:00010000:1.0:1787826300.435533:0:67244:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x203:0x0]) in obj [0x200000404:0x203:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826300.435558:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826300.435576:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1787826300.435586:0:67244:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x203:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826300.435594:0:67244:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) 00000080:00010000:1.0:1787826300.435598:0:67244:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) for lock 0xe5f736940357b8ea 00000080:00010000:1.0:1787826300.435632:0:67244:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b8ea from it ffff9dd141f63cd8 00010000:00010000:1.0:1787826300.435637:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826300.435786:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1787826300.435816:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/1, layout version 8 restore needed 0 00020000:00200000:1.0:1787826300.435830:0:67244:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826300.435833:0:67244:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826300.435839:0:67244:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x203:0x0] 00000008:00010000:1.0:1787826300.435864:0:67244:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826300.435869:0:67244:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c32fb0, osc lock ffff907d962f5998, flags 0x400000400001002 00010000:00010000:1.0:1787826300.435890:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826300.435903:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00200000:1.0:1787826300.436046:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826300.436050:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826300.436060:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000020:00200000:1.0:1787826300.436135:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826300.436143:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:1.0:1787826300.436147:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5791148 (ffff907d884a7ea8/0) at cl_lock_release():245 00000020:00010000:1.0:1787826300.436151:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5791148 (ffff907d884a7ea8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826300.436154:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c32fb0 (ffff907d884a7b48/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826300.436159:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5791148 (ffff907d884a7ea8/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826300.436162:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c32fb0 (ffff907d884a7ea8/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826300.436174:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/1, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826300.436194:0:67244:0:(file.c:2652:do_file_write_iter()) START file f204c.sanity-flr:[0x200000404:0x203:0x0], ppos: 1048576, count: 1048576 00000080:00200000:1.0:1787826300.436202:0:67244:0:(file.c:2090:ll_file_io_generic()) f204c.sanity-flr: write ppos: 1048576, bytes: 1048576 00000080:00200000:1.0:1787826300.436211:0:67244:0:(file.c:1848:ll_io_set_mirror()) f204c.sanity-flr: desiginated mirror: 2 00000020:00200000:1.0:1787826300.436218:0:67244:0:(cl_io.c:218:cl_io_rw_init()) header@000000008ae07a1f[0x110, 2, [0x200000404:0x203:0x0] hash] 00000020:00200000:1.0:1787826300.436220:0:67244:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [1048576, 2097152) 2048 0 00000080:00200000:1.0:1787826300.436223:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 8 restore needed 0 00000080:00200000:1.0:1787826300.436236:0:67244:0:(file.c:2170:ll_file_io_generic()) Range lock [256, 511] 00020000:00200000:1.0:1787826300.436244:0:67244:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 1048576 chunk: [1048576, 2097152) 2097152, 1048576 00020000:00200000:1.0:1787826300.436248:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[3] flags 0x11 00000080:00200000:1.0:1787826300.436263:0:67244:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [256, 511] 00000008:00010000:1.0:1787826300.436271:0:67244:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 11 00000008:00010000:1.0:1787826300.436275:0:67244:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c32fb0, osc lock ffff907d962f5998, flags 0x40000 00000080:00200000:1.0:1787826300.436296:0:67244:0:(vvp_io.c:1339:vvp_io_write_start()) f204c.sanity-flr: write [1048576, 2097152) append 0 append lockpos 0 00000080:00200000:1.0:1787826300.436312:0:67244:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908), size=1048576 (max 335544320), offset=1048576=0x100000, pages 256 (max 81920), lockless, parallel 00000020:00200000:1.0:1787826300.436594:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:1.0:1787826300.436597:0:67244:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 2097152 00000080:00200000:1.0:1787826300.436631:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:1.0:1787826300.436636:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98af23f8 (ffff907d884a7ea8/0) at cl_lock_release():245 00000020:00010000:1.0:1787826300.436638:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98af23f8 (ffff907d884a7ea8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826300.436641:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c32fb0 (ffff907d884a7b48/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826300.436718:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98af23f8 (ffff907d884a7ea8/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826300.436721:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c32fb0 (ffff907d884a7ea8/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826300.437023:0:67244:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:1.0:1787826300.437031:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1787826300.437061:0:58785:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907d908aa000 00000008:00100000:1.0:1787826300.441046:0:58785:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d908a1e00 - object 0x280000400:780 offset 1048576<>2097152 00000100:00100000:0.0:1787826300.441259:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011431680:192.168.203.104@tcp:4:lfs.0 00000100:00100000:0.0:1787826300.470715:0:2348:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0001-osc-ffff907d986d0000: committing for last_committed 17179873841 gen 1 00000008:00010000:0.0:1787826300.470808:0:2348:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907d908aa000 00000020:00200000:0.0:1787826300.470819:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:0.0:1787826300.470822:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:0.0:1787826300.470824:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:0.0:1787826300.470826:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:0.0:1787826300.470828:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:0.0:1787826300.470829:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:0.0:1787826300.470831:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:0.0:1787826300.470833:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:0.0:1787826300.470836:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:0.0:1787826300.470838:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:0.0:1787826300.470840:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:0.0:1787826300.470841:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:0.0:1787826300.470843:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:0.0:1787826300.470845:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:0.0:1787826300.470846:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:0.0:1787826300.470848:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:0.0:1787826300.470850:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:0.0:1787826300.470851:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:0.0:1787826300.470853:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:0.0:1787826300.470867:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:0.0:1787826300.470870:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:0.0:1787826300.470872:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:0.0:1787826300.470874:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:0.0:1787826300.470876:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:0.0:1787826300.470877:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:0.0:1787826300.470878:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:0.0:1787826300.470880:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:0.0:1787826300.470882:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:0.0:1787826300.470884:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:0.0:1787826300.470885:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:0.0:1787826300.470887:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:0.0:1787826300.470888:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:0.0:1787826300.470891:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:0.0:1787826300.470893:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:0.0:1787826300.470894:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:0.0:1787826300.470896:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:0.0:1787826300.470898:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:0.0:1787826300.470899:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:0.0:1787826300.470901:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:0.0:1787826300.470912:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:0.0:1787826300.470915:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:0.0:1787826300.470916:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:0.0:1787826300.470918:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:0.0:1787826300.470920:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:0.0:1787826300.470921:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:0.0:1787826300.470923:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:0.0:1787826300.470925:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:0.0:1787826300.470927:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:0.0:1787826300.470929:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:0.0:1787826300.470930:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:0.0:1787826300.470932:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:0.0:1787826300.470933:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:0.0:1787826300.470935:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:0.0:1787826300.470937:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:0.0:1787826300.470939:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:0.0:1787826300.470941:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:0.0:1787826300.470943:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:0.0:1787826300.470945:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:0.0:1787826300.470947:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:0.0:1787826300.470948:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:0.0:1787826300.470950:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:0.0:1787826300.470952:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:0.0:1787826300.470953:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:0.0:1787826300.470955:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:0.0:1787826300.470956:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:0.0:1787826300.470958:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:0.0:1787826300.470959:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:0.0:1787826300.470961:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:0.0:1787826300.470962:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:0.0:1787826300.470964:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:0.0:1787826300.470965:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:0.0:1787826300.470967:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:0.0:1787826300.470969:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:0.0:1787826300.470971:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:0.0:1787826300.470972:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:0.0:1787826300.470974:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:0.0:1787826300.470976:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:0.0:1787826300.470977:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:0.0:1787826300.470979:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:0.0:1787826300.470980:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:0.0:1787826300.470981:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:0.0:1787826300.470983:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:0.0:1787826300.470985:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:0.0:1787826300.470987:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:0.0:1787826300.470989:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:0.0:1787826300.470991:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:0.0:1787826300.470992:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:0.0:1787826300.470994:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:0.0:1787826300.470996:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:0.0:1787826300.470997:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:0.0:1787826300.470998:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:0.0:1787826300.471000:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:0.0:1787826300.471001:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:0.0:1787826300.471002:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:0.0:1787826300.471003:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:0.0:1787826300.471005:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:0.0:1787826300.471006:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:0.0:1787826300.471008:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:0.0:1787826300.471009:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:0.0:1787826300.471011:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:0.0:1787826300.471013:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:0.0:1787826300.471015:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:0.0:1787826300.471016:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:0.0:1787826300.471018:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:0.0:1787826300.471020:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:0.0:1787826300.471022:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:0.0:1787826300.471024:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:0.0:1787826300.471026:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:0.0:1787826300.471027:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:0.0:1787826300.471029:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:0.0:1787826300.471030:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:0.0:1787826300.471032:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:0.0:1787826300.471033:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:0.0:1787826300.471035:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:0.0:1787826300.471037:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:0.0:1787826300.471039:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:0.0:1787826300.471040:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:0.0:1787826300.471042:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:0.0:1787826300.471044:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:0.0:1787826300.471046:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:0.0:1787826300.471047:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:0.0:1787826300.471049:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:0.0:1787826300.471051:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:0.0:1787826300.471053:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:0.0:1787826300.471055:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:0.0:1787826300.471057:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:0.0:1787826300.471060:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:0.0:1787826300.471061:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:0.0:1787826300.471063:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:0.0:1787826300.471065:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:0.0:1787826300.471066:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:0.0:1787826300.471068:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:0.0:1787826300.471069:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:0.0:1787826300.471228:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:0.0:1787826300.471231:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:0.0:1787826300.471233:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:0.0:1787826300.471236:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:0.0:1787826300.471238:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:0.0:1787826300.471240:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:0.0:1787826300.471242:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:0.0:1787826300.471244:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:0.0:1787826300.471246:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:0.0:1787826300.471248:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:0.0:1787826300.471250:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:0.0:1787826300.471251:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:0.0:1787826300.471253:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:0.0:1787826300.471255:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:0.0:1787826300.471257:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:0.0:1787826300.471259:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:0.0:1787826300.471261:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:0.0:1787826300.471263:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:0.0:1787826300.471265:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:0.0:1787826300.471266:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:0.0:1787826300.471268:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:0.0:1787826300.471270:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:0.0:1787826300.471272:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:0.0:1787826300.471273:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:0.0:1787826300.471275:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:0.0:1787826300.471276:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:0.0:1787826300.471278:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:0.0:1787826300.471290:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:0.0:1787826300.471292:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:0.0:1787826300.471294:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:0.0:1787826300.471296:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:0.0:1787826300.471297:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:0.0:1787826300.471299:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:0.0:1787826300.471301:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:0.0:1787826300.471303:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:0.0:1787826300.471305:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:0.0:1787826300.471307:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:0.0:1787826300.471309:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:0.0:1787826300.471311:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:0.0:1787826300.471313:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:0.0:1787826300.471315:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:0.0:1787826300.471317:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:0.0:1787826300.471319:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:0.0:1787826300.471321:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:0.0:1787826300.471323:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:0.0:1787826300.471325:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:0.0:1787826300.471327:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:0.0:1787826300.471328:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:0.0:1787826300.471330:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:0.0:1787826300.471332:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:0.0:1787826300.471334:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:0.0:1787826300.471336:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:0.0:1787826300.471338:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:0.0:1787826300.471340:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:0.0:1787826300.471342:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:0.0:1787826300.471343:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:0.0:1787826300.471345:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:0.0:1787826300.471347:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:0.0:1787826300.471348:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:0.0:1787826300.471350:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:0.0:1787826300.471352:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:0.0:1787826300.471353:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:0.0:1787826300.471355:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:0.0:1787826300.471357:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:0.0:1787826300.471358:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:0.0:1787826300.471360:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:0.0:1787826300.471362:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:0.0:1787826300.471364:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:0.0:1787826300.471366:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:0.0:1787826300.471368:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:0.0:1787826300.471369:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:0.0:1787826300.471371:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:0.0:1787826300.471373:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:0.0:1787826300.471374:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:0.0:1787826300.471376:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:0.0:1787826300.471378:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:0.0:1787826300.471379:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:0.0:1787826300.471381:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:0.0:1787826300.471383:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:0.0:1787826300.471385:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:0.0:1787826300.471387:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:0.0:1787826300.471389:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:0.0:1787826300.471390:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:0.0:1787826300.471392:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:0.0:1787826300.471394:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:0.0:1787826300.471396:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:0.0:1787826300.471397:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:0.0:1787826300.471399:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:0.0:1787826300.471401:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:0.0:1787826300.471403:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:0.0:1787826300.471405:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:0.0:1787826300.471407:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:0.0:1787826300.471409:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:0.0:1787826300.471411:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:0.0:1787826300.471413:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:0.0:1787826300.471414:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:0.0:1787826300.471416:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:0.0:1787826300.471418:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:0.0:1787826300.471420:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:0.0:1787826300.471423:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:0.0:1787826300.471425:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:0.0:1787826300.471427:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:0.0:1787826300.471429:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:0.0:1787826300.471431:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:0.0:1787826300.471432:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:0.0:1787826300.471434:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:0.0:1787826300.471435:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:0.0:1787826300.471437:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:0.0:1787826300.471439:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:0.0:1787826300.471441:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:0.0:1787826300.471443:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:0.0:1787826300.471445:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:0.0:1787826300.471447:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:0.0:1787826300.471449:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:0.0:1787826300.471451:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:0.0:1787826300.471452:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:0.0:1787826300.471454:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:0.0:1787826300.471456:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:0.0:1787826300.471458:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:0.0:1787826300.471460:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:0.0:1787826300.471461:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:0.0:1787826300.471463:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:0.0:1787826300.471464:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1787826300.471572:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1787826300.472204:0:2348:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907d908aa000 00000100:00100000:0.0:1787826300.472243:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011431680:192.168.203.104@tcp:4:lfs.0 00000080:00200000:2.0:1787826300.476193:0:67244:0:(file.c:2207:ll_file_io_generic()) Range unlock [256, 511] 00000080:00200000:2.0:1787826300.476220:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826300.476228:0:67244:0:(file.c:2240:ll_file_io_generic()) f204c.sanity-flr: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1787826300.476244:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000020:00200000:2.0:1787826300.476250:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826300.476269:0:67244:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200080:2.0:1787826300.476308:0:67244:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908) cmd=400866b1 arg=0 00000080:00200080:2.0:1787826300.476320:0:67244:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908) cmd=400866b1 arg=1 00000080:00200008:2.0:1787826300.476329:0:67244:0:(file.c:6313:ll_getattr_dentry()) START file f204c.sanity-flr:[0x200000404:0x203:0x0](ffff907d93310908), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1787826300.476338:0:67244:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908),name=f204c.sanity-flr 00000002:00010000:2.0:1787826300.476366:0:67244:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x203:0x0]) in obj [0x200000404:0x203:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826300.476392:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826300.476410:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1787826300.476420:0:67244:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x203:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826300.476430:0:67244:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) 00000080:00010000:2.0:1787826300.476435:0:67244:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) for lock 0xe5f736940357b8ea 00000080:00010000:2.0:1787826300.476444:0:67244:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b8ea from it ffff9dd141f63cd8 00010000:00010000:2.0:1787826300.476449:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1787826300.476459:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1787826300.476488:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/1, layout version 8 restore needed 0 00020000:00200000:2.0:1787826300.476501:0:67244:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826300.476505:0:67244:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1787826300.476510:0:67244:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x203:0x0] 00000008:00010000:2.0:1787826300.476536:0:67244:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826300.476540:0:67244:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322030, osc lock ffff907d830a29c0, flags 0x400000400001002 00010000:00010000:2.0:1787826300.476561:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1787826300.476580:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000020:00200000:2.0:1787826300.476592:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826300.476595:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1787826300.476603:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000020:00200000:2.0:1787826300.478525:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826300.478546:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826300.478553:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5791148 (ffff907d884a7ea8/0) at cl_lock_release():245 00000020:00010000:2.0:1787826300.478557:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5791148 (ffff907d884a7ea8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826300.478562:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322030 (ffff907d884a7b48/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826300.478567:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5791148 (ffff907d884a7ea8/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826300.478570:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322030 (ffff907d884a7ea8/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826300.478591:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/1, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826300.478635:0:67244:0:(file.c:5224:ll_file_seek()) START file f204c.sanity-flr:[0x200000404:0x203:0x0], offset: 2097152, type: SEEK_DATA 00000080:00200000:2.0:1787826300.478645:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 8 restore needed 0 00020000:00200000:2.0:1787826300.478655:0:67244:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826300.478658:0:67244:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1787826300.478665:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1787826300.478676:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:2.0:1787826300.478694:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000080:00200000:2.0:1787826300.478704:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826300.478711:0:67244:0:(file.c:1848:ll_io_set_mirror()) f204c.sanity-flr: desiginated mirror: 1 00000080:00200000:2.0:1787826300.478714:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1787826300.478720:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1787826300.478723:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:2.0:1787826300.478727:0:67244:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [512, 18446744073709551615] 00000008:00010000:2.0:1787826300.478743:0:67244:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:2.0:1787826300.478747:0:67244:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322030, osc lock ffff907d830a29c0, flags 0x0 00010000:00010000:2.0:1787826300.478772:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1787826300.478788:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (2097152 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1787826300.478803:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1787826300.478812:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1787826300.478821:0:67244:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1787826300.478832:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826300.478838:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98af23f8 (ffff907d884a7ea8/0) at cl_lock_release():245 00000020:00010000:2.0:1787826300.478840:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98af23f8 (ffff907d884a7ea8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826300.478843:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322030 (ffff907d884a7b48/1) at cl_lock_cancel():142 00010000:00010000:2.0:1787826300.478847:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1787826300.478874:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00000020:00010000:2.0:1787826300.478884:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98af23f8 (ffff907d884a7ea8/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826300.478887:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322030 (ffff907d884a7ea8/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826300.478894:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826300.478899:0:67244:0:(file.c:5262:ll_file_seek()) COMPLETED file f204c.sanity-flr:[0x200000404:0x203:0x0], offset: 2097152, type: SEEK_DATA, rc = -6 00000080:00200008:2.0:1787826300.480012:0:67244:0:(file.c:5224:ll_file_seek()) START file f204c.sanity-flr:[0x200000404:0x203:0x0], offset: 0, type: (null) 00000080:00200000:2.0:1787826300.480029:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/1, layout version 8 restore needed 0 00020000:00200000:2.0:1787826300.480038:0:67244:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826300.480042:0:67244:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1787826300.480045:0:67244:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x203:0x0] 00000008:00010000:2.0:1787826300.480061:0:67244:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826300.480065:0:67244:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322030, osc lock ffff907d830a29c0, flags 0x400000400001002 00010000:00010000:2.0:1787826300.480159:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1787826300.480178:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00000020:00200000:2.0:1787826300.480191:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826300.480195:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1787826300.480204:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00000020:00200000:2.0:1787826300.480218:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826300.480226:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826300.480230:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5791148 (ffff907d884a7ea8/0) at cl_lock_release():245 00000020:00010000:2.0:1787826300.480233:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5791148 (ffff907d884a7ea8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826300.480236:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322030 (ffff907d884a7b48/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826300.480240:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5791148 (ffff907d884a7ea8/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826300.480243:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322030 (ffff907d884a7ea8/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826300.480255:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/1, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826300.480262:0:67244:0:(file.c:5262:ll_file_seek()) COMPLETED file f204c.sanity-flr:[0x200000404:0x203:0x0], offset: 0, type: (null), rc = 2097152 00000080:00200080:2.0:1787826300.480274:0:67244:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908) cmd=400866b1 arg=0 00000080:00200080:2.0:1787826300.497404:0:67244:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908) cmd=400866b1 arg=2 00000080:00200008:2.0:1787826300.497432:0:67244:0:(file.c:6313:ll_getattr_dentry()) START file f204c.sanity-flr:[0x200000404:0x203:0x0](ffff907d93310908), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1787826300.497443:0:67244:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908),name=f204c.sanity-flr 00000002:00010000:2.0:1787826300.497470:0:67244:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x203:0x0]) in obj [0x200000404:0x203:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826300.497492:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1787826300.497510:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1787826300.497520:0:67244:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x203:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826300.497530:0:67244:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) 00000080:00010000:2.0:1787826300.497534:0:67244:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) for lock 0xe5f736940357b8ea 00000080:00010000:2.0:1787826300.497543:0:67244:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b8ea from it ffff9dd141f63cd8 00010000:00010000:2.0:1787826300.497548:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1787826300.497559:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1787826300.497594:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/1, layout version 8 restore needed 0 00020000:00200000:2.0:1787826300.497712:0:67244:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826300.497716:0:67244:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1787826300.497722:0:67244:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x203:0x0] 00000008:00010000:2.0:1787826300.497748:0:67244:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826300.497753:0:67244:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322030, osc lock ffff907d830a29c0, flags 0x400000400001002 00010000:00010000:2.0:1787826300.497773:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1787826300.497786:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00000020:00200000:2.0:1787826300.497800:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826300.497804:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1787826300.497813:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00000020:00200000:2.0:1787826300.497825:0:67244:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826300.497837:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826300.497842:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5791148 (ffff907d884a7ea8/0) at cl_lock_release():245 00000020:00010000:2.0:1787826300.497846:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5791148 (ffff907d884a7ea8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826300.497849:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322030 (ffff907d884a7b48/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826300.497865:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5791148 (ffff907d884a7ea8/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826300.497869:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322030 (ffff907d884a7ea8/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826300.497882:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/1, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826300.498039:0:67244:0:(llite_lib.c:2427:ll_setattr_raw()) START file f204c.sanity-flr:[0x200000404:0x203:0x0](ffff907d93310908) current size 2097152, valid attrs 2068, mode 0, uid -1976721614, gid -1, new size 2097152, atime 1787826300.000000000, mtime 1787826300.000000000, ctime 1787826300.000000000 00010000:00010000:2.0:1787826300.498182:0:67244:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1787826300.498203:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826300.498260:0:67244:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 1/0,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1787826300.498268:0:67244:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1787826300.498271:0:67244:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ae00/0xe5f736940357b8ea lrc: 0/0,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc10056 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 7 lru_type: 0 00000020:00100000:2.0:1787826300.498307:0:67244:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1787826300.498322:0:67244:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d91148b40 x1874670011431936/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787826300.498353:0:67244:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011431936:192.168.203.104@tcp:36:lfs.0 00000100:00100000:2.0:1787826300.498560:0:67244:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d888673c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826300.503048:0:67244:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011431936:192.168.203.104@tcp:36:lfs.0 00000080:00200000:0.0:1787826300.503120:0:67244:0:(file.c:1848:ll_io_set_mirror()) f204c.sanity-flr: desiginated mirror: 2 00000080:00200000:0.0:1787826300.503130:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826300.503158:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[3] flags 0x11 00000080:00200000:0.0:1787826300.503214:0:67244:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:0.0:1787826300.503252:0:67244:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1787826300.503259:0:67244:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d84f913f0, osc lock ffff907d96368618, flags 0x0 00000008:00010000:0.0:1787826300.503263:0:67244:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1787826300.503266:0:67244:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d84f91430, osc lock ffff907d96369860, flags 0x0 00010000:00010000:0.0:1787826300.503357:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b8f8 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30b:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.503373:0:67244:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b8f8 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30b:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.503385:0:67244:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b8f8 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30b:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826300.503410:0:67244:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011432064:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826300.503506:0:67244:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f91cc0 going to sleep for 11 seconds 00000100:00100000:0.0:1787826300.506482:0:67244:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011432064:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826300.506522:0:67244:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b8f8 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x30b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10080 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.506547:0:67244:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b8f8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10080 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.506562:0:67244:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b8f8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10080 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.506581:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b8f8 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x30b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10080 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826300.506603:0:67244:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 1048576, setting rss=1048576; kms=1048576, end=18446744073709551615 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b8f8 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x30b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10080 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.506999:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b8f8 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x30b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10080 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.507008:0:67244:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b8f8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10080 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.507161:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b8ff lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30c:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.507171:0:67244:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b8ff lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30c:0x0].0x0 rrc: 2 type: EXT [1048576->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.507182:0:67244:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b8ff lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30c:0x0].0x0 rrc: 2 type: EXT [1048576->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826300.507207:0:67244:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011432192:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826300.507364:0:67244:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f91cc0 going to sleep for 11 seconds 00000100:00100000:0.0:1787826300.510063:0:67244:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011432192:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826300.510158:0:67244:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b8ff lrc: 4/0,1 mode: --/PW res: [0x280000400:0x30c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10087 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.510188:0:67244:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b8ff lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10087 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.510203:0:67244:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b8ff lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10087 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.510220:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b8ff lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x30c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10087 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826300.510235:0:67244:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b8ff lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x30c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10087 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.510249:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b8ff lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x30c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10087 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.510257:0:67244:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b8ff lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10087 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826300.510419:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011432320:192.168.203.104@tcp:10:lfs.0 00000100:00100000:1.0:1787826300.513572:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011432320:192.168.203.104@tcp:10:lfs.0 00000080:00200000:0.0:1787826300.513664:0:67244:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x203:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:0.0:1787826300.513674:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98af2e48 (ffff907d884a75a8/0) at cl_lock_release():245 00000020:00010000:0.0:1787826300.513678:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98af2e48 (ffff907d884a75a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1787826300.513684:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d84f913f0 (ffff907d884a7518/1) at cl_lock_cancel():142 00010000:00010000:0.0:1787826300.513697:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b8f8 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10080 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.513716:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b8f8 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x30b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10080 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:0.0:1787826300.513913:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d84f91430 (ffff907d884a7368/1) at cl_lock_cancel():142 00010000:00010000:0.0:1787826300.513921:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b8ff lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10087 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.513932:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b8ff lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x30c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10087 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:0.0:1787826300.513941:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98af2e48 (ffff907d884a75a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1787826300.513944:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d84f913f0 (ffff907d884a75a8/1) at cl_lock_fini():72 00000020:00010000:0.0:1787826300.513950:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d84f91430 (ffff907d884a75a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1787826300.513977:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1787826300.513994:0:67244:0:(llite_lib.c:2686:ll_setattr_raw()) COMPLETED file f204c.sanity-flr:[0x200000404:0x203:0x0](ffff907d93310908) current size 2097152, valid attrs 10002168, mode 0, uid -1976721614, gid -1, new size 2097152, atime 1787826300.000000000, mtime 1787826300.000000000, ctime 1787826300.000000000, rc 0 00000080:00200080:0.0:1787826300.514020:0:67244:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908) cmd=400866b1 arg=0 00000080:00200008:0.0:1787826300.514056:0:67244:0:(llite_lib.c:2427:ll_setattr_raw()) START file f204c.sanity-flr:[0x200000404:0x203:0x0](ffff907d93310908) current size 2097152, valid attrs 101f0, mode 9dd1, uid -1786300736, gid -28547, new size 18446621467747617032, atime 1787826275.000000000, mtime 1787826287.000000000, ctime 1787826300.000000000 00000020:00100000:0.0:1787826300.514186:0:67244:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1787826300.514199:0:67244:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908470c0 x1874670011432576/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787826300.514228:0:67244:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011432576:192.168.203.104@tcp:36:lfs.0 00000100:00100000:0.0:1787826300.514410:0:67244:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f913c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826300.518101:0:67244:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011432576:192.168.203.104@tcp:36:lfs.0 00000080:00200000:0.0:1787826300.518163:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 8 restore needed 0 00020000:00200000:0.0:1787826300.518180:0:67244:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826300.518184:0:67244:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826300.518193:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826300.518206:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:0.0:1787826300.518210:0:67244:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:0.0:1787826300.518228:0:67244:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1787826300.518234:0:67244:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da032ecb0, osc lock ffff907d96369860, flags 0x0 00010000:00010000:0.0:1787826300.518255:0:67244:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d91e6e400 type 11 mode 2 res 9663677440/806 (0 10485759) 00010000:00010000:0.0:1787826300.518327:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.518342:0:67244:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.518354:0:67244:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826300.518380:0:67244:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011432704:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826300.519360:0:67244:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f913c0 going to sleep for 11 seconds 00000100:00100000:3.0:1787826300.521918:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826300.522046:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025495296 00000100:00100000:2.0:1787826300.522058:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025495296 00000100:00100000:2.0:1787826300.522199:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7147 00000100:00100000:2.0:1787826300.522211:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83525e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:28560:x1874670025495296:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826300.522239:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1787826300.522414:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826300.522422:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83525e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:28560:x1874670025495296:12345-192.168.203.104@tcp:104: Request processed in 214us (582us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826300.522431:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7147 00000100:00100000:2.0:1787826300.524670:0:67244:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011432704:192.168.203.104@tcp:101:lfs.0 00010000:00010000:2.0:1787826300.524711:0:67244:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.524742:0:67244:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.524752:0:67244:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.525058:0:60482:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826300.525068:0:60482:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00010000:00010000:0.0:1787826300.525139:0:60482:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883b000 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826300.525146:0:60482:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800428400000000 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00000080:00200000:0.0:1787826300.525220:0:60482:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1787826300.525237:0:60482:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1787826300.525310:0:60482:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 2/0,0 mode: --/PR res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00010000:00010000:0.0:1787826300.525318:0:60482:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:1.0:1787826300.525391:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011432832:192.168.203.104@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1787826300.525539:0:60482:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00010000:00010000:0.0:1787826300.525549:0:60482:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007a58bcfe/0xe5f736940357b8f1 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc1005d expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00010000:00010000:0.0:1787826300.525569:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826300.525571:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826300.525573:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826300.527867:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826300.527974:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025495424 00000100:00100000:2.0:1787826300.527985:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025495424 00000100:00100000:2.0:1787826300.528056:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7148 00000100:00100000:2.0:1787826300.528068:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83524000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:5933:x1874670025495424:12345-192.168.203.104@tcp:105: 00010000:00010000:2.0:1787826300.528096:0:50445:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 5/0,1 mode: --/PW res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.528112:0:50445:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 5/0,1 mode: --/PW res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.528130:0:50445:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 5/0,1 mode: --/PW res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.528143:0:50445:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10080000020000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.528154:0:50445:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10080000020000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.528165:0:50445:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x80000020000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.528194:0:50445:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff907d8883a000) 00000100:00100000:2.0:1787826300.528264:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83524000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:5933:x1874670025495424:12345-192.168.203.104@tcp:105: Request processed in 199us (422us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826300.528273:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7148 00010000:00010000:0.0:1787826300.528692:0:67244:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.528706:0:67244:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.528735:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826300.528752:0:67244:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10020000020000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.528767:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.528776:0:67244:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826300.530719:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011432960:192.168.203.104@tcp:2:lfs.0 00000100:00100000:1.0:1787826300.533142:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011432832:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826300.533550:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011432960:192.168.203.104@tcp:2:lfs.0 00000020:00010000:2.0:1787826300.533594:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98af2e48 (ffff907d884a75a8/0) at cl_lock_release():245 00000020:00010000:2.0:1787826300.533600:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98af2e48 (ffff907d884a75a8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826300.533677:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da032ecb0 (ffff907d884a7518/1) at cl_lock_cancel():142 00010000:00010000:2.0:1787826300.533693:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.533714:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1787826300.533821:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98af2e48 (ffff907d884a75a8/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826300.533825:0:67244:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da032ecb0 (ffff907d884a75a8/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826300.533851:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 0/0, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826300.533943:0:67244:0:(llite_lib.c:2686:ll_setattr_raw()) COMPLETED file f204c.sanity-flr:[0x200000404:0x203:0x0](ffff907d93310908) current size 2097152, valid attrs 100101f0, mode 9dd1, uid -1786300736, gid -28547, new size 18446621467747617032, atime 1787826275.000000000, mtime 1787826287.000000000, ctime 1787826300.000000000, rc 0 00000080:00200080:2.0:1787826300.533972:0:67244:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908) cmd=c00c66f3 arg=16c12a0 00010000:00010000:2.0:1787826300.534042:0:67244:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b8dc lrc: 2/0,0 mode: PW/PW res: [0x200000404:0x203:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x81008000000000 nid: local remote: 0x7918213f4fc1002c expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.534064:0:67244:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357b8dc lrc: 0/0,0 mode: --/PW res: [0x200000404:0x203:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x85809400000000 nid: local remote: 0x7918213f4fc1002c expref: -99 pid: 67244 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826300.534157:0:67244:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d90844f00 x1874670011430016/t8589942986(8589942986) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 968/1056 e 0 to 0 dl 1787826318 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1787826300.534168:0:67244:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1787826300.534174:0:67244:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d91148780 x1874670011433216/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 504/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1787826300.534201:0:67244:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011433216:192.168.203.104@tcp:35:lfs.0 00000100:00100000:2.0:1787826300.534370:0:67244:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88867f00 going to sleep for 11 seconds 00000100:00100000:3.0:1787826300.537364:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826300.537754:0:40339:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:2.0:1787826300.537781:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025495552 00000100:00100000:2.0:1787826300.537793:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025495552 00000100:00100000:2.0:1787826300.537967:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7149 00000100:00100000:2.0:1787826300.537981:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d88635c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32460:x1874670025495552:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826300.538007:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826300.538110:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826300.538118:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d88635c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32460:x1874670025495552:12345-192.168.203.104@tcp:104: Request processed in 139us (767us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826300.538127:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7149 00010000:00010000:1.0:1787826300.538208:0:60835:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826300.538218:0:60835:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826300.538229:0:60835:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883ac00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826300.538235:0:60835:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826300.538251:0:60835:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826300.538260:0:60835:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 3/0,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1787826300.538300:0:60835:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x203:0x0]: losing layout lock 00010000:00010000:1.0:1787826300.538354:0:60835:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 2/0,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826300.538362:0:60835:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826300.538395:0:60835:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 1/0,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826300.538404:0:60835:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b8e3 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc10048 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826300.538423:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826300.538425:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826300.538427:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826300.538545:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1787826300.538567:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011433344:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826300.546167:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011433344:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826300.548427:0:67244:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589942991 gen 1 00000100:00100000:0.0:1787826300.548464:0:67244:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011433216:192.168.203.104@tcp:35:lfs.0 00000002:00100000:0.0:1787826300.548502:0:67244:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d90844f00 x1874670011430016/t8589942986(8589942986) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 968/1056 e 0 to 0 dl 1787826318 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:0.0:1787826300.548543:0:67244:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 31 res 8589935620/515 (0 0) 00000080:00010000:0.0:1787826300.548557:0:67244:0:(file.c:7076:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000404:0x203:0x0](ffff907d93310908) 00000002:00010000:0.0:1787826300.548570:0:67244:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x203:0x0]) in obj [0x200000404:0x203:0x0], intent: layout flags 0 00010000:00010000:0.0:1787826300.548671:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 3/1,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67244 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.548687:0:67244:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 3/1,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67244 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.548698:0:67244:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 3/1,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67244 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826300.548722:0:67244:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011433472:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826300.548821:0:67244:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f91cc0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826300.556926:0:67244:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011433472:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826300.556969:0:67244:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc100a3 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.556988:0:67244:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc100a3 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826300.557006:0:67244:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x400 disposition=0x1, status=0 req@ffff907d90844f00 x1874670011433472/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/792 e 0 to 0 dl 1787826318 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826300.557017:0:67244:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 448 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc100a3 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1787826300.557036:0:67244:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc100a3 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.557051:0:67244:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/515 (0 0) 00000080:00200000:0.0:1787826300.557059:0:67244:0:(namei.c:952:ll_intent_lock()) intent lock 1024 on i1 [0x200000404:0x203:0x0] suppgids 0 0: rc 0 00000080:00010000:0.0:1787826300.557197:0:67244:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) for lock 0xe5f736940357b90d 00000080:00010000:0.0:1787826300.557209:0:67244:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b90d from it ffff9dd141f63bc8 00010000:00010000:0.0:1787826300.557214:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc100a3 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.557226:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc100a3 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826300.557255:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc100a3 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826300.557268:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc100a3 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:0.0:1787826300.557277:0:67244:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x203:0x0](00000000010d8bed) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc100a3 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826300.557342:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc100a3 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826300.557351:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc100a3 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826300.557370:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826300.557384:0:67244:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826300.557388:0:67244:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826300.557397:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826300.557445:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:0.0:1787826300.557472:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1787826300.557486:0:67244:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc100a3 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826300.557496:0:67244:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc100a3 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1787826300.557504:0:67244:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x203:0x0](00000000010d8bed) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc100a3 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826300.557522:0:67244:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826300.557527:0:67244:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826300.557529:0:67244:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826300.557532:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826300.557551:0:67244:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:0.0:1787826300.557558:0:67244:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:0.0:1787826300.557572:0:67244:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 2097152 blocks 4096 [cma]time: 1787826287 1787826287 0 by oinfo size 2097152 blocks 4096 [cma]time 1787826300 1787826287 1787826275 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x326:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:0.0:1787826300.557767:0:67244:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 1048576 blocks 2048 [cma]time: 1787826300 1787826300 0 by oinfo size 1048576 blocks 2048 [cma]time 1787826300 1787826300 0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b8f8 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x30b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10080 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:1.0:1787826300.558394:0:67244:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 2097152 blocks 2048 [cma]time: 1787826300 1787826300 0 by oinfo size 2097152 blocks 2048 [cma]time 1787826300 1787826300 0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b8ff lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x30c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10087 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826300.558484:0:67244:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x203:0x0]: layout version change: 4294967294 -> 9 00010000:00010000:1.0:1787826300.558492:0:67244:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc100a3 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826300.558508:0:67244:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc100a3 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:1.0:1787826300.558556:0:67244:0:(file.c:479:ll_file_release()) START file f204c.sanity-flr:[0x200000404:0x203:0x0](ffff907d93310908), flags 154002 00000002:00100000:1.0:1787826300.558717:0:67244:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d83527840 x1874670011429760/t8589942985(8589942985) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 968/1056 e 0 to 0 dl 1787826318 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826300.558787:0:67244:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826300.558795:0:67244:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a2d00 x1874670011433728/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826300.558825:0:67244:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011433728:192.168.203.104@tcp:35:lfs.0 00000100:00100000:1.0:1787826300.559306:0:67244:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d96147000 going to sleep for 11 seconds 00000100:00100000:1.0:1787826300.561825:0:67244:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67244:1874670011433728:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826300.562026:0:67244:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d83527840 x1874670011429760/t8589942985(8589942985) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 968/1056 e 0 to 0 dl 1787826318 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1787826300.574000:0:67245:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826300.574039:0:67245:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826300.574064:0:67245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00010000:00010000:2.0:1787826300.574150:0:67245:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:2.0:1787826300.574160:0:67245:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826300.574171:0:67245:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826300.574175:0:67245:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:2.0:1787826300.574184:0:67245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141ccbbf8 00010000:00010000:2.0:1787826300.574189:0:67245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00010000:00010000:2.0:1787826300.574199:0:67245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:2.0:1787826300.574220:0:67245:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826300.574229:0:67245:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=27 00000080:00200000:2.0:1787826300.574237:0:67245:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 1 00000080:00200000:2.0:1787826300.574250:0:67245:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), file ffff907d91edf200, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1787826300.574259:0:67245:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), intent=open 00000002:00010000:2.0:1787826300.574278:0:67245:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x202:0x0], intent: open flags 0304001 00010000:00010000:2.0:1787826300.574365:0:67245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c0400/0xe5f736940357b914 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 67245 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.574374:0:67245:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c0400/0xe5f736940357b914 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67245 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826300.574389:0:67245:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826300.574405:0:67245:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d911483c0 x1874670011433984/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826300.574417:0:67245:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c0400/0xe5f736940357b914 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67245 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826300.574444:0:67245:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67245:1874670011433984:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826300.574958:0:67245:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88867900 going to sleep for 13 seconds 00000100:00100000:0.0:1787826300.579311:0:67245:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67245:1874670011433984:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826300.579344:0:67245:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c0400/0xe5f736940357b914 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67245 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.579362:0:67245:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c0400/0xe5f736940357b914 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 67245 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.579374:0:67245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c0400/0xe5f736940357b914 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 67245 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.579381:0:67245:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c0400/0xe5f736940357b914 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 67245 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.579392:0:67245:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c0400/0xe5f736940357b914 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 67245 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.579399:0:67245:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d863c0400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826300.579405:0:67245:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c0400/0xe5f736940357b914 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 67245 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.579413:0:67245:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c0400/0xe5f736940357b914 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 67245 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.579421:0:67245:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c0400/0xe5f736940357b914 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 67245 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.579430:0:67245:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826300.579439:0:67245:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c0400/0xe5f736940357b914 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 67245 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.579447:0:67245:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c0400/0xe5f736940357b914 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 67245 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826300.579473:0:67245:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d911483c0 x1874670011433984/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826318 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826300.579489:0:67245:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x202:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1787826300.580338:0:67245:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826300.580505:0:67245:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826300.580526:0:67245:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:0.0:1787826300.580543:0:67245:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:0.0:1787826300.580552:0:67245:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826300.580560:0:67245:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826300.580564:0:67245:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:0.0:1787826300.580571:0:67245:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141ccbbf8 00010000:00010000:0.0:1787826300.580576:0:67245:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:0.0:1787826300.580584:0:67245:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:0.0:1787826300.580601:0:67245:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826300.581025:0:67245:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=267 00000080:00200000:0.0:1787826300.581037:0:67245:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 44 00000080:00200008:0.0:1787826300.581049:0:67245:0:(file.c:1090:ll_file_open()) START file d204c.sanity-flr:[0x200000404:0x202:0x0](ffff907d93313248), flags 304000 00000080:00200080:0.0:1787826300.581185:0:67245:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248) cmd=c0086915 arg=8493d0 00000100:00100000:0.0:1787826300.581273:0:67245:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67245:1874670011434112:192.168.203.104@tcp:34:lfs.0 00000100:00100000:0.0:1787826300.581459:0:67245:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f91cc0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826300.585715:0:67245:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67245:1874670011434112:192.168.203.104@tcp:34:lfs.0 00000080:00200008:0.0:1787826300.593522:0:67245:0:(file.c:479:ll_file_release()) START file d204c.sanity-flr:[0x200000404:0x202:0x0](ffff907d93313248), flags 304000 00010000:00010000:0.0:1787826300.593575:0:67245:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b8b2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x202:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff76 expref: -99 pid: 67231 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200008:2.0:1787826300.607759:0:67247:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1787826300.607772:0:67247:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826300.607797:0:67247:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826300.607822:0:67247:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1787826300.607840:0:67247:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:2.0:1787826300.607850:0:67247:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826300.607860:0:67247:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826300.607864:0:67247:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:2.0:1787826300.607872:0:67247:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd140e33cb8 00010000:00010000:2.0:1787826300.607877:0:67247:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1787826300.607887:0:67247:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:2.0:1787826300.608000:0:67247:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826300.608008:0:67247:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826300.608015:0:67247:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1787826300.608025:0:67247:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:2.0:1787826300.608031:0:67247:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826300.608035:0:67247:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826300.608038:0:67247:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:2.0:1787826300.608042:0:67247:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd140e33bb0 00010000:00010000:2.0:1787826300.608045:0:67247:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1787826300.608053:0:67247:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:2.0:1787826300.608063:0:67247:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:2.0:1787826300.608071:0:67247:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:2.0:1787826300.608091:0:67247:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffe5a5536ac 00000080:00200008:2.0:1787826300.608105:0:67247:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:0.0:1787826300.609355:0:67251:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826300.609375:0:67251:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826300.609389:0:67251:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:0.0:1787826300.609403:0:67251:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:0.0:1787826300.609411:0:67251:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826300.609417:0:67251:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826300.609421:0:67251:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:0.0:1787826300.609428:0:67251:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141f7bbf8 00010000:00010000:0.0:1787826300.609431:0:67251:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:0.0:1787826300.609441:0:67251:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:0.0:1787826300.609453:0:67251:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826300.609461:0:67251:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=27 00000080:00200000:0.0:1787826300.609466:0:67251:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 1 00000080:00200000:0.0:1787826300.609477:0:67251:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), file ffff907d83962c00, open_flags 18800, mode 0 opened 0 00000080:00200000:0.0:1787826300.609484:0:67251:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), intent=open 00000002:00010000:0.0:1787826300.609500:0:67251:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x202:0x0], intent: open flags 0304001 00010000:00010000:0.0:1787826300.609563:0:67251:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b91b lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 67251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.609577:0:67251:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b91b lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67251 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826300.609589:0:67251:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826300.609603:0:67251:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90844f00 x1874670011434368/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826300.609637:0:67251:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b91b lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67251 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826300.609663:0:67251:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67251:1874670011434368:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826300.611316:0:67251:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f91cc0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826300.615165:0:67251:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67251:1874670011434368:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826300.615197:0:67251:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b91b lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.615213:0:67251:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b91b lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 67251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.615221:0:67251:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b91b lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 67251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.615229:0:67251:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b91b lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 67251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.615239:0:67251:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b91b lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 67251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.615246:0:67251:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883b400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826300.615253:0:67251:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b91b lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 67251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.615261:0:67251:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b91b lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 67251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.615270:0:67251:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b91b lrc: 5/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 67251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.615277:0:67251:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826300.615294:0:67251:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b91b lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 67251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.615301:0:67251:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b91b lrc: 0/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 67251 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826300.615324:0:67251:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d90844f00 x1874670011434368/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826318 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826300.615340:0:67251:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x202:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826300.615948:0:67251:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826300.615976:0:67251:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826300.615995:0:67251:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:1.0:1787826300.616011:0:67251:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:1.0:1787826300.616019:0:67251:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826300.616025:0:67251:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826300.616030:0:67251:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:1.0:1787826300.616038:0:67251:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141f7bbf8 00010000:00010000:1.0:1787826300.616041:0:67251:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:1.0:1787826300.616049:0:67251:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:1.0:1787826300.616065:0:67251:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826300.616087:0:67251:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=267 00000080:00200000:1.0:1787826300.616096:0:67251:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 44 00000080:00200008:1.0:1787826300.616105:0:67251:0:(file.c:1090:ll_file_open()) START file d204c.sanity-flr:[0x200000404:0x202:0x0](ffff907d93313248), flags 304000 00000080:00200080:1.0:1787826300.616148:0:67251:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248) cmd=c1406916 arg=d1da20 00000100:00100000:1.0:1787826300.616224:0:67251:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67251:1874670011434496:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826300.616403:0:67251:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d96147000 going to sleep for 13 seconds 00000100:00100000:1.0:1787826300.619780:0:67251:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67251:1874670011434496:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826300.619984:0:67251:0:(file.c:479:ll_file_release()) START file d204c.sanity-flr:[0x200000404:0x202:0x0](ffff907d93313248), flags 304000 00010000:00010000:1.0:1787826300.620022:0:67251:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b8b2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x202:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff76 expref: -99 pid: 67231 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200008:2.0:1787826300.643532:0:67254:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1787826300.643543:0:67254:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826300.643565:0:67254:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826300.643586:0:67254:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:2.0:1787826300.643599:0:67254:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:2.0:1787826300.643727:0:67254:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826300.643737:0:67254:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826300.643742:0:67254:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:2.0:1787826300.643751:0:67254:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141e57cb8 00010000:00010000:2.0:1787826300.643755:0:67254:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:2.0:1787826300.643766:0:67254:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00000080:00200000:2.0:1787826300.643950:0:67254:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826300.643960:0:67254:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826300.643968:0:67254:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00010000:00010000:2.0:1787826300.643978:0:67254:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00000080:00200000:2.0:1787826300.643985:0:67254:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826300.643990:0:67254:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826300.643992:0:67254:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:2.0:1787826300.643997:0:67254:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141e57bb0 00010000:00010000:2.0:1787826300.644000:0:67254:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00010000:00010000:2.0:1787826300.644007:0:67254:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000080:00200000:2.0:1787826300.644017:0:67254:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:2.0:1787826300.644024:0:67254:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:2.0:1787826300.644045:0:67254:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffcb3a610bc 00000080:00200008:2.0:1787826300.644060:0:67254:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:0.0:1787826300.646020:0:67256:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826300.646047:0:67256:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826300.646079:0:67256:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:0.0:1787826300.646098:0:67256:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000080:00200000:0.0:1787826300.646108:0:67256:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826300.646117:0:67256:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826300.646121:0:67256:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:0.0:1787826300.646131:0:67256:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141f8bbf8 00010000:00010000:0.0:1787826300.646135:0:67256:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:0.0:1787826300.646144:0:67256:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00000080:00200000:0.0:1787826300.646160:0:67256:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826300.646168:0:67256:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=27 00000080:00200000:0.0:1787826300.646174:0:67256:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 1 00000080:00200000:0.0:1787826300.646187:0:67256:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), file ffff907d83963e00, open_flags 18800, mode 0 opened 0 00000080:00200000:0.0:1787826300.646194:0:67256:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), intent=open 00000002:00010000:0.0:1787826300.646209:0:67256:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x202:0x0], intent: open flags 0304001 00010000:00010000:0.0:1787826300.646271:0:67256:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b922 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 67256 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.646290:0:67256:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b922 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67256 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826300.646304:0:67256:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826300.646320:0:67256:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90844f00 x1874670011434752/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826300.646331:0:67256:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b922 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67256 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826300.646355:0:67256:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67256:1874670011434752:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826300.646586:0:67256:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f91cc0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826300.652645:0:67256:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67256:1874670011434752:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826300.652679:0:67256:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b922 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67256 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.652698:0:67256:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b922 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 67256 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.652706:0:67256:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b922 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 67256 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.652713:0:67256:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b922 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 67256 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.652723:0:67256:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b922 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 67256 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.652740:0:67256:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883b400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826300.652746:0:67256:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b922 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 67256 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.652754:0:67256:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b922 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 67256 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.652762:0:67256:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b922 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 67256 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.652770:0:67256:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826300.652778:0:67256:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b922 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 67256 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826300.652785:0:67256:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b922 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 67256 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826300.652814:0:67256:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d90844f00 x1874670011434752/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826318 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826300.652831:0:67256:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x202:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1787826300.653146:0:67256:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826300.653172:0:67256:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826300.653191:0:67256:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00010000:00010000:0.0:1787826300.653204:0:67256:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00000080:00200000:0.0:1787826300.653212:0:67256:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826300.653218:0:67256:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826300.653223:0:67256:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:0.0:1787826300.653229:0:67256:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141f8bbf8 00010000:00010000:0.0:1787826300.653233:0:67256:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00010000:00010000:0.0:1787826300.653242:0:67256:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00000080:00200000:0.0:1787826300.653261:0:67256:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826300.653271:0:67256:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=267 00000080:00200000:0.0:1787826300.653292:0:67256:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 44 00000080:00200008:0.0:1787826300.653303:0:67256:0:(file.c:1090:ll_file_open()) START file d204c.sanity-flr:[0x200000404:0x202:0x0](ffff907d93313248), flags 304000 00000080:00200080:0.0:1787826300.653351:0:67256:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248) cmd=c1406916 arg=1812a20 00000100:00100000:0.0:1787826300.653424:0:67256:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67256:1874670011434880:192.168.203.104@tcp:34:lfs.0 00000100:00100000:0.0:1787826300.653545:0:67256:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d84f91cc0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826300.663152:0:67256:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67256:1874670011434880:192.168.203.104@tcp:34:lfs.0 00000080:00200008:3.0:1787826300.663434:0:67256:0:(file.c:479:ll_file_release()) START file d204c.sanity-flr:[0x200000404:0x202:0x0](ffff907d93313248), flags 304000 00010000:00010000:3.0:1787826300.663476:0:67256:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b8b2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x202:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff76 expref: -99 pid: 67231 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200008:1.0:1787826300.694529:0:67261:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1787826300.694544:0:67261:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826300.694573:0:67261:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826300.694597:0:67261:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:1.0:1787826300.694912:0:67261:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00000080:00200000:1.0:1787826300.694923:0:67261:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826300.694933:0:67261:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826300.694938:0:67261:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:1.0:1787826300.694949:0:67261:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141ccbcb8 00010000:00010000:1.0:1787826300.694953:0:67261:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:1.0:1787826300.694963:0:67261:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00000080:00200000:1.0:1787826300.695115:0:67261:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826300.695124:0:67261:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826300.695133:0:67261:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00010000:00010000:1.0:1787826300.695143:0:67261:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00000080:00200000:1.0:1787826300.695150:0:67261:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826300.695154:0:67261:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826300.695156:0:67261:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:1.0:1787826300.695160:0:67261:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141ccbbb0 00010000:00010000:1.0:1787826300.695164:0:67261:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00010000:00010000:1.0:1787826300.695171:0:67261:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00000080:00200000:1.0:1787826300.695181:0:67261:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:1.0:1787826300.695189:0:67261:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:1.0:1787826300.695208:0:67261:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffc213c351c 00000080:00200008:1.0:1787826300.695224:0:67261:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:2.0:1787826300.696925:0:67264:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826300.696953:0:67264:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826300.696973:0:67264:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00010000:00010000:2.0:1787826300.696988:0:67264:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00000080:00200000:2.0:1787826300.696997:0:67264:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826300.697004:0:67264:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826300.697008:0:67264:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:2.0:1787826300.697015:0:67264:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141f7bbf8 00010000:00010000:2.0:1787826300.697019:0:67264:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00010000:00010000:2.0:1787826300.697028:0:67264:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00000080:00200000:2.0:1787826300.697043:0:67264:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826300.697051:0:67264:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=27 00000080:00200000:2.0:1787826300.697058:0:67264:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 1 00000080:00200000:2.0:1787826300.697071:0:67264:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), file ffff907d83528480, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1787826300.697078:0:67264:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), intent=open 00000002:00010000:2.0:1787826300.697095:0:67264:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x202:0x0], intent: open flags 0304001 00010000:00010000:2.0:1787826300.697162:0:67264:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1000/0xe5f736940357b929 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 67264 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.697171:0:67264:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1000/0xe5f736940357b929 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67264 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826300.697185:0:67264:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826300.697199:0:67264:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114b480 x1874670011435136/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826300.697209:0:67264:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1000/0xe5f736940357b929 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67264 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826300.697234:0:67264:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67264:1874670011435136:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826300.697689:0:67264:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88867900 going to sleep for 13 seconds 00000100:00100000:2.0:1787826300.701672:0:67264:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67264:1874670011435136:192.168.203.104@tcp:101:lfs.0 00010000:00010000:2.0:1787826300.701703:0:67264:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1000/0xe5f736940357b929 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67264 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.701720:0:67264:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1000/0xe5f736940357b929 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 67264 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.701727:0:67264:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1000/0xe5f736940357b929 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 67264 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.701736:0:67264:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1000/0xe5f736940357b929 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 67264 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.701746:0:67264:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1000/0xe5f736940357b929 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 67264 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.701753:0:67264:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d863c1000 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:2.0:1787826300.701758:0:67264:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1000/0xe5f736940357b929 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 67264 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.701765:0:67264:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1000/0xe5f736940357b929 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 67264 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.701773:0:67264:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1000/0xe5f736940357b929 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 67264 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.701780:0:67264:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1787826300.701788:0:67264:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1000/0xe5f736940357b929 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 67264 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826300.701795:0:67264:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1000/0xe5f736940357b929 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 67264 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826300.701815:0:67264:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d9114b480 x1874670011435136/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826318 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1787826300.701832:0:67264:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x202:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1787826300.703320:0:67264:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826300.703355:0:67264:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826300.703376:0:67264:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00010000:00010000:2.0:1787826300.703393:0:67264:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00000080:00200000:2.0:1787826300.703401:0:67264:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826300.703408:0:67264:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826300.703412:0:67264:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:2.0:1787826300.703419:0:67264:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141f7bbf8 00010000:00010000:2.0:1787826300.703423:0:67264:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00010000:00010000:2.0:1787826300.703432:0:67264:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00000080:00200000:2.0:1787826300.703447:0:67264:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826300.703457:0:67264:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=267 00000080:00200000:2.0:1787826300.703464:0:67264:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 44 00000080:00200008:2.0:1787826300.703474:0:67264:0:(file.c:1090:ll_file_open()) START file d204c.sanity-flr:[0x200000404:0x202:0x0](ffff907d93313248), flags 304000 00000080:00200080:2.0:1787826300.703516:0:67264:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248) cmd=c1406916 arg=155ca20 00000100:00100000:2.0:1787826300.703595:0:67264:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67264:1874670011435264:192.168.203.104@tcp:34:lfs.0 00000100:00100000:2.0:1787826300.705136:0:67264:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88867900 going to sleep for 13 seconds 00000100:00100000:3.0:1787826300.714769:0:67264:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67264:1874670011435264:192.168.203.104@tcp:34:lfs.0 00000080:00200008:3.0:1787826300.714998:0:67264:0:(file.c:479:ll_file_release()) START file d204c.sanity-flr:[0x200000404:0x202:0x0](ffff907d93313248), flags 304000 00010000:00010000:3.0:1787826300.715037:0:67264:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b8b2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x202:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff76 expref: -99 pid: 67231 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1787826300.740921:0:67267:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826300.741042:0:67267:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826300.741072:0:67267:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00010000:00010000:1.0:1787826300.741094:0:67267:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00000080:00200000:1.0:1787826300.741103:0:67267:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826300.741113:0:67267:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826300.741117:0:67267:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:1.0:1787826300.741127:0:67267:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141de3bf8 00010000:00010000:1.0:1787826300.741131:0:67267:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00010000:00010000:1.0:1787826300.741142:0:67267:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00000080:00200000:1.0:1787826300.741161:0:67267:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826300.741171:0:67267:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=27 00000080:00200000:1.0:1787826300.741177:0:67267:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 1 00000080:00200000:1.0:1787826300.741345:0:67267:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), file ffff907d83b1f680, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1787826300.741355:0:67267:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), intent=open 00000002:00010000:1.0:1787826300.741378:0:67267:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x202:0x0], intent: open flags 0304001 00010000:00010000:1.0:1787826300.741451:0:67267:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b930 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 67267 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826300.741459:0:67267:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b930 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67267 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826300.741474:0:67267:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826300.741589:0:67267:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a2d00 x1874670011435520/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826300.741603:0:67267:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b930 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67267 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826300.741755:0:67267:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67267:1874670011435520:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826300.742798:0:67267:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d96147000 going to sleep for 13 seconds 00000100:00100000:3.0:1787826300.747723:0:67267:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67267:1874670011435520:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826300.747771:0:67267:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b930 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67267 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826300.747788:0:67267:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b930 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 67267 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826300.747796:0:67267:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b930 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 67267 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826300.747804:0:67267:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b930 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 67267 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826300.747814:0:67267:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b930 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 67267 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826300.747822:0:67267:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0b400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826300.747829:0:67267:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b930 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 67267 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826300.747837:0:67267:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b930 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 67267 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826300.747845:0:67267:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b930 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 67267 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826300.747853:0:67267:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826300.747861:0:67267:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b930 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 67267 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826300.747867:0:67267:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357b930 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 67267 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826300.748155:0:67267:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908a2d00 x1874670011435520/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826318 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826300.748173:0:67267:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x202:0x0] suppgids 0 -1: rc -20 00000080:00200000:3.0:1787826300.750924:0:67267:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826300.751042:0:67267:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826300.751068:0:67267:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00010000:00010000:3.0:1787826300.751089:0:67267:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00000080:00200000:3.0:1787826300.751097:0:67267:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826300.751106:0:67267:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826300.751110:0:67267:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:3.0:1787826300.751119:0:67267:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141de3bf8 00010000:00010000:3.0:1787826300.751125:0:67267:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00010000:00010000:3.0:1787826300.751135:0:67267:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00000080:00200000:3.0:1787826300.751154:0:67267:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826300.751164:0:67267:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=267 00000080:00200000:3.0:1787826300.751173:0:67267:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 44 00000080:00200008:3.0:1787826300.751184:0:67267:0:(file.c:1090:ll_file_open()) START file d204c.sanity-flr:[0x200000404:0x202:0x0](ffff907d93313248), flags 304000 00000080:00200080:3.0:1787826300.751313:0:67267:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248) cmd=c0086915 arg=1a6c3d0 00000100:00100000:3.0:1787826300.751404:0:67267:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835243c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67267:1874670011435648:192.168.203.104@tcp:34:lfs.0 00000100:00100000:3.0:1787826300.751831:0:67267:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d870bb300 going to sleep for 13 seconds 00000100:00100000:3.0:1787826300.758875:0:67267:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835243c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:67267:1874670011435648:192.168.203.104@tcp:34:lfs.0 00000080:00200008:3.0:1787826300.759512:0:67267:0:(file.c:479:ll_file_release()) START file d204c.sanity-flr:[0x200000404:0x202:0x0](ffff907d93313248), flags 304000 00010000:00010000:3.0:1787826300.759554:0:67267:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ba00/0xe5f736940357b8b2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x202:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0ff76 expref: -99 pid: 67231 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:3.0:1787826300.790287:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826301.815773:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826302.709837:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011435776:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826302.709964:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011436032:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826302.710735:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1787826302.710748:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011435904:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826302.710802:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011436160:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826302.716276:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011435776:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826302.716307:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011436032:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826302.723343:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011435904:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826302.723374:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011436160:192.168.203.104@tcp:400:kworker.0 00010000:00010000:0.0:1787826302.851890:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826303.867848:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826304.889185:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826305.912324:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1787826306.438293:0:67683:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826306.438320:0:67683:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826306.438338:0:67683:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00010000:00010000:0.0:1787826306.438354:0:67683:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00000080:00200000:0.0:1787826306.438362:0:67683:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826306.438368:0:67683:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826306.438372:0:67683:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:0.0:1787826306.438380:0:67683:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141f7bb58 00010000:00010000:0.0:1787826306.438384:0:67683:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00010000:00010000:0.0:1787826306.438393:0:67683:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00000080:00200000:0.0:1787826306.438408:0:67683:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826306.438416:0:67683:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=24 00000080:00200000:0.0:1787826306.438421:0:67683:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 1 00000080:00200000:0.0:1787826306.438432:0:67683:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), flags=8 00000080:00200000:0.0:1787826306.438436:0:67683:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248), intent=getattr 00000002:00010000:0.0:1787826306.438448:0:67683:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x202:0x0], intent: getattr flags 01 00010000:00010000:0.0:1787826306.438492:0:67683:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 67683 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826306.438500:0:67683:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67683 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826306.438510:0:67683:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67683 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826306.438533:0:67683:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:67683:1874670011436288:192.168.203.104@tcp:101:rm.0 00000100:00100000:0.0:1787826306.438923:0:67683:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd9c0 going to sleep for 13 seconds 00000100:00100000:2.0:1787826306.462973:0:67683:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589942992 gen 1 00000100:00100000:2.0:1787826306.462989:0:67683:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:67683:1874670011436288:192.168.203.104@tcp:101:rm.0 00010000:00010000:2.0:1787826306.463011:0:67683:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x202:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10191 expref: -99 pid: 67683 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826306.463026:0:67683:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10191 expref: -99 pid: 67683 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826306.463034:0:67683:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10191 expref: -99 pid: 67683 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826306.463044:0:67683:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10191 expref: -99 pid: 67683 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826306.463055:0:67683:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10191 expref: -99 pid: 67683 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826306.463067:0:67683:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10191 expref: -99 pid: 67683 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826306.463075:0:67683:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10191 expref: -99 pid: 67683 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826306.463083:0:67683:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10191 expref: -99 pid: 67683 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826306.463094:0:67683:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d90847840 x1874670011436288/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/1032 e 0 to 0 dl 1787826324 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1787826306.463107:0:67683:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 448 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10191 expref: -99 pid: 67683 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1787826306.463120:0:67683:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10191 expref: -99 pid: 67683 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826306.463128:0:67683:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/515 (0 0) 00000080:00200000:2.0:1787826306.463134:0:67683:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x202:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1787826306.463187:0:67683:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x203:0x0], updating i_size 2097152 00000080:00200000:2.0:1787826306.463191:0:67683:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x203:0x0](ffff907d93310908): rc = 0 00000080:00200000:2.0:1787826306.463205:0:67683:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x203:0x0]: layout version change: 9 -> 9 00000080:00010000:2.0:1787826306.463211:0:67683:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) for lock 0xe5f736940357b937 00000080:00010000:2.0:1787826306.463221:0:67683:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) 00000080:00010000:2.0:1787826306.463224:0:67683:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b937 from it ffff9dd141f7bb20 00010000:00010000:2.0:1787826306.463228:0:67683:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10191 expref: -99 pid: 67683 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826306.463236:0:67683:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10191 expref: -99 pid: 67683 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1787826306.463283:0:67683:0:(file.c:6313:ll_getattr_dentry()) START file f204c.sanity-flr:[0x200000404:0x203:0x0](ffff907d93310908), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1787826306.463290:0:67683:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908),name=f204c.sanity-flr 00000002:00010000:2.0:1787826306.463308:0:67683:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x203:0x0]) in obj [0x200000404:0x202:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826306.463319:0:67683:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10191 expref: -99 pid: 67683 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826306.463329:0:67683:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10191 expref: -99 pid: 67683 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826306.463336:0:67683:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x202:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826306.463340:0:67683:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) 00000080:00010000:2.0:1787826306.463342:0:67683:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x203:0x0](ffff907d93310908) for lock 0xe5f736940357b937 00000080:00010000:2.0:1787826306.463346:0:67683:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b937 from it ffff9dd141f7bcb8 00010000:00010000:2.0:1787826306.463348:0:67683:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10191 expref: -99 pid: 67683 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826306.463355:0:67683:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10191 expref: -99 pid: 67683 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1787826306.463399:0:67683:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826306.463407:0:67683:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826306.463415:0:67683:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00010000:00010000:2.0:1787826306.463422:0:67683:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00000080:00200000:2.0:1787826306.463427:0:67683:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826306.463430:0:67683:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826306.463432:0:67683:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:2.0:1787826306.463436:0:67683:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd141f7bc10 00010000:00010000:2.0:1787826306.463439:0:67683:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00010000:00010000:2.0:1787826306.463444:0:67683:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00000080:00200000:2.0:1787826306.463453:0:67683:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826306.463458:0:67683:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=24 00000080:00200000:2.0:1787826306.463461:0:67683:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 1 00000080:00200000:2.0:1787826306.463467:0:67683:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204c.sanity-flr, flags=0 00000080:00200000:2.0:1787826306.463470:0:67683:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x202:0x0](ffff907d93313248), inode mode 41ed mask 3 00000080:00200000:2.0:1787826306.463475:0:67683:0:(namei.c:2418:ll_unlink()) VFS Op:name=f204c.sanity-flr, dir=[0x200000404:0x202:0x0](ffff907d93313248) 00010000:00010000:2.0:1787826306.463498:0:67683:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc100a3 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1787826306.463512:0:67683:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x203:0x0]: losing layout lock 00010000:00010000:2.0:1787826306.463523:0:67683:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc10191 expref: -99 pid: 67683 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1787826306.463531:0:67683:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x203:0x0]: losing layout lock 00010000:00010000:2.0:1787826306.463538:0:67683:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b896 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x203:0x0].0x0 bits 0x20/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc0ff14 expref: -99 pid: 67203 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826306.463582:0:67683:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 1/0,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc100a3 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826306.463590:0:67683:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc10191 expref: -99 pid: 67683 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826306.463596:0:67683:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b896 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0ff14 expref: -99 pid: 67203 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826306.463602:0:67683:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1787826306.463628:0:67683:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b90d lrc: 0/0,0 mode: --/CR res: [0x200000404:0x203:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc100a3 expref: -99 pid: 67244 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826306.468415:0:67683:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b400/0xe5f736940357b937 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc10191 expref: -99 pid: 67683 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826306.468426:0:67683:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357b896 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x203:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0ff14 expref: -99 pid: 67203 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000020:00100000:2.0:1787826306.468455:0:67683:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1787826306.468467:0:67683:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d91149e00 x1874670011436544/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 512/2200 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787826306.468796:0:67683:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149e00 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:67683:1874670011436544:192.168.203.104@tcp:36:rm.0 00000100:00100000:2.0:1787826306.468885:0:67683:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d96147000 going to sleep for 13 seconds 00000100:00100000:0.0:1787826306.490948:0:67683:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149e00 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:67683:1874670011436544:192.168.203.104@tcp:36:rm.0 00000080:00200000:0.0:1787826306.491012:0:67683:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x203:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826306.491025:0:67683:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x203:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826306.491028:0:67683:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826306.491034:0:67683:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826306.491044:0:67683:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:0.0:1787826306.491063:0:67683:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x203:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1787826306.491072:0:67683:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200000404:0x203:0x0](ffff907d93310908) 00010000:00010000:0.0:1787826306.935102:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826307.830564:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011436672:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826307.830811:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011436928:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826307.832310:0:2351:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1787826307.832332:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011436800:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826307.832388:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011437056:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826307.841796:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011436800:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826307.841813:0:2351:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0001-osc-ffff907d986d0000: committing for last_committed 17179873842 gen 1 00000100:00100000:2.0:1787826307.841833:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011437056:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826307.841843:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011436672:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826307.841855:0:2351:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff907d986d0000: committing for last_committed 21474840504 gen 3 00000100:00100000:2.0:1787826307.841863:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011436928:192.168.203.104@tcp:400:kworker.0 00010000:00010000:1.0:1787826307.959821:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1787826308.289021:0:67824:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-flr test 204d: FLR write/stale/resync sel test with repeated comp ========================================================== 06:25:06 (1787826306) 00010000:00010000:3.0:1787826308.980917:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826310.006387:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826310.539070:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826310.539231:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025498368 00000100:00100000:3.0:1787826310.539245:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025498368 00000100:00100000:3.0:1787826310.539320:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7150 00000100:00100000:3.0:1787826310.539332:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d9114bc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:9562:x1874670025498368:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826310.539358:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x420000030000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826310.539474:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826310.539481:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d9114bc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:9562:x1874670025498368:12345-192.168.203.104@tcp:104: Request processed in 152us (427us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826310.539489:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7150 00010000:00010000:0.0:1787826310.541677:0:40347:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826310.541696:0:40347:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x420000030000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826310.541712:0:40347:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a000 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826310.541720:0:40347:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x428400030000 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826310.541796:0:40347:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826310.541805:0:40347:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826310.541855:0:40347:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826310.541864:0:40347:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000b702346a/0xe5f736940357b906 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x326:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc10095 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826310.541894:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826310.541897:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826310.541898:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826310.542042:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011437184:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1787826310.543136:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826310.547022:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025498496 00000100:00100000:2.0:1787826310.547041:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025498496 00000100:00100000:2.0:1787826310.547114:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7151 00000100:00100000:2.0:1787826310.547126:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83524f00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:35506:x1874670025498496:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826310.547151:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b8f8 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x30b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc10080 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826310.547259:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826310.547266:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83524f00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:35506:x1874670025498496:12345-192.168.203.104@tcp:104: Request processed in 143us (4137us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826310.547274:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7151 00010000:00010000:0.0:1787826310.547341:0:61297:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826310.547351:0:61297:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b8f8 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x30b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc10080 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826310.547361:0:61297:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a200 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826310.547368:0:61297:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b8f8 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x30b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x428400010000 nid: local remote: 0x7918213f4fc10080 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826310.547428:0:61297:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b8f8 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x30b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc10080 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826310.547436:0:61297:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826310.547458:0:61297:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b8f8 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x30b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc10080 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826310.547468:0:61297:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357b8f8 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x30b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc10080 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826310.547514:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826310.547516:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826310.547519:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826310.547537:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011437312:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826310.547559:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011437184:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1787826310.551389:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826310.551735:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025498752 00000100:00100000:2.0:1787826310.551747:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025498752 00000100:00100000:2.0:1787826310.551891:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7152 00000100:00100000:2.0:1787826310.551906:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83525a40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:27403:x1874670025498752:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826310.551932:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b8ff lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x30c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc10087 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826310.552199:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826310.552207:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83525a40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:27403:x1874670025498752:12345-192.168.203.104@tcp:104: Request processed in 305us (833us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826310.552229:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7152 00000100:00100000:1.0:1787826310.553293:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011437312:192.168.203.104@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1787826310.556876:0:47852:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826310.556895:0:47852:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b8ff lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x30c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc10087 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826310.556914:0:47852:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883ba00 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826310.556924:0:47852:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b8ff lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x30c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x428400010000 nid: local remote: 0x7918213f4fc10087 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826310.557343:0:47852:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b8ff lrc: 2/0,0 mode: --/PW res: [0x280000400:0x30c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc10087 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826310.557354:0:47852:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826310.557391:0:47852:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b8ff lrc: 1/0,0 mode: --/PW res: [0x280000400:0x30c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc10087 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826310.557401:0:47852:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b6dabee8/0xe5f736940357b8ff lrc: 0/0,0 mode: --/PW res: [0x280000400:0x30c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc10087 expref: -99 pid: 67244 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826310.557433:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826310.557435:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826310.557437:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826310.557447:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1787826310.557486:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011437440:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826310.570054:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011437440:192.168.203.104@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1787826311.030035:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826312.052855:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826312.958976:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011437696:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826312.959108:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011437952:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826312.959153:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1787826312.959159:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011437568:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826312.959183:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011437824:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826312.967693:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011437568:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826312.967723:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011437824:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826312.967736:0:2349:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589942993 gen 1 00000100:00100000:0.0:1787826312.967759:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011437696:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826312.967773:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011437952:192.168.203.104@tcp:400:kworker.0 00010000:00010000:2.0:1787826313.076862:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826314.102414:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826315.124787:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826316.148777:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826317.173664:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826318.070232:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011438208:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826318.070343:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011438464:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826318.070380:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1787826318.070387:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011438080:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826318.070412:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011438336:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826318.074314:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011438080:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826318.074333:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011438336:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826318.074344:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011438208:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826318.074352:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011438464:192.168.203.104@tcp:400:kworker.0 00010000:00010000:1.0:1787826318.199690:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826319.221098:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826320.244963:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826321.271039:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826322.296736:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826323.189069:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011438592:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826323.189293:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011438848:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826323.189412:0:2351:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1787826323.189423:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011438720:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826323.189456:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011438976:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826323.200552:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011438720:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826323.200575:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011438976:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826323.200599:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011438592:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826323.200651:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011438848:192.168.203.104@tcp:400:kworker.0 00010000:00010000:3.0:1787826323.317144:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826324.340862:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1787826324.642060:0:68247:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826324.642099:0:68247:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826324.642135:0:68247:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00010000:00010000:2.0:1787826324.642155:0:68247:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00000080:00200000:2.0:1787826324.642164:0:68247:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826324.642174:0:68247:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826324.642179:0:68247:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:2.0:1787826324.642189:0:68247:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd142ad7be0 00010000:00010000:2.0:1787826324.642195:0:68247:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00010000:00010000:2.0:1787826324.642205:0:68247:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00000080:00200000:2.0:1787826324.642233:0:68247:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826324.642266:0:68247:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d204d.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), flags=1536 00000080:00200000:2.0:1787826324.642270:0:68247:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826324.642276:0:68247:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826324.642283:0:68247:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00010000:00010000:2.0:1787826324.642292:0:68247:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00000080:00200000:2.0:1787826324.642300:0:68247:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826324.642304:0:68247:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826324.642306:0:68247:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:2.0:1787826324.642310:0:68247:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd142ad7ce0 00010000:00010000:2.0:1787826324.642313:0:68247:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00010000:00010000:2.0:1787826324.642322:0:68247:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00000080:00200000:2.0:1787826324.642331:0:68247:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 3 00000080:00200000:2.0:1787826324.642338:0:68247:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826324.642344:0:68247:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826324.642349:0:68247:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00010000:00010000:2.0:1787826324.642364:0:68247:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00000080:00200000:2.0:1787826324.642371:0:68247:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826324.642374:0:68247:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826324.642377:0:68247:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b850 00000080:00010000:2.0:1787826324.642380:0:68247:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b850 from it ffff9dd142ad7dc8 00010000:00010000:2.0:1787826324.642383:0:68247:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00010000:00010000:2.0:1787826324.642391:0:68247:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00000080:00200000:2.0:1787826324.642398:0:68247:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 3 00000080:00200000:2.0:1787826324.642404:0:68247:0:(namei.c:2239:do_mkdir()) VFS Op:name=d204d.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148) 00800000:00000004:2.0:1787826324.642435:0:68247:0:(obd_class.h:1056:obd_statfs_async()) lustre-MDT0000-mdc-ffff907d986d0000: age 1629, max_age 0 00010000:00010000:2.0:1787826324.642515:0:68247:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00010000:00010000:2.0:1787826324.642573:0:68247:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00010000:00010000:2.0:1787826324.642581:0:68247:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1787826324.642585:0:68247:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b850 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc0fe26 expref: -99 pid: 67180 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00000020:00100000:2.0:1787826324.642647:0:68247:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1787826324.642664:0:68247:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d91149680 x1874670011439360/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 568/2152 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787826324.642691:0:68247:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job mkdir:6f891bcb-15cd-41e0-96e2-db843619c1be:68247:1874670011439360:192.168.203.104@tcp:36:mkdir.0 00000100:00100000:3.0:1787826324.642976:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011439104:192.168.203.104@tcp:41:mkdir.0 00000100:00100000:2.0:1787826324.642997:0:68247:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88d28900 going to sleep for 13 seconds 00000100:00100000:3.0:1787826324.647024:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011439104:192.168.203.104@tcp:41:mkdir.0 00000100:00100000:0.0:1787826324.647179:0:68247:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job mkdir:6f891bcb-15cd-41e0-96e2-db843619c1be:68247:1874670011439360:192.168.203.104@tcp:36:mkdir.0 00000080:00200000:0.0:1787826324.647238:0:68247:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908) 00000080:00200000:0.0:1787826324.647246:0:68247:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x205:0x0], updating i_size 4096 00000080:00200000:0.0:1787826324.707747:0:68250:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826324.707835:0:68250:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826324.707923:0:68250:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.707935:0:68250:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.707946:0:68250:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826324.707980:0:68250:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68250:1874670011439488:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826324.708369:0:68250:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd300 going to sleep for 13 seconds 00000100:00100000:0.0:1787826324.718538:0:68250:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68250:1874670011439488:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826324.718574:0:68250:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.718589:0:68250:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.718603:0:68250:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.718652:0:68250:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.718666:0:68250:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.718676:0:68250:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.718684:0:68250:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826324.718696:0:68250:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d90844000 x1874670011439488/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/688 e 0 to 0 dl 1787826342 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826324.718710:0:68250:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.718797:0:68250:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1787826324.718808:0:68250:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1787826324.718843:0:68250:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:0.0:1787826324.718851:0:68250:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:0.0:1787826324.718858:0:68250:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826324.718862:0:68250:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142267b58 00010000:00010000:0.0:1787826324.718866:0:68250:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.718876:0:68250:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826324.718906:0:68250:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826324.718932:0:68250:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d204d.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), flags=29 00000080:00200000:0.0:1787826324.718937:0:68250:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d204d.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=0 00000002:00010000:0.0:1787826324.718961:0:68250:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: d204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00010000:00010000:0.0:1787826324.719008:0:68250:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b945 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.719016:0:68250:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b945 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.719027:0:68250:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b945 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826324.719052:0:68250:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68250:1874670011439616:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826324.719229:0:68250:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd300 going to sleep for 13 seconds 00000100:00100000:0.0:1787826324.731966:0:68250:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68250:1874670011439616:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826324.732004:0:68250:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b945 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101fa expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.732037:0:68250:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b945 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101fa expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.732047:0:68250:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b945 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x205:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101fa expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.732061:0:68250:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b945 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101fa expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.732074:0:68250:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b945 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101fa expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.732089:0:68250:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b945 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101fa expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.732097:0:68250:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b945 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101fa expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.732105:0:68250:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b945 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101fa expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826324.732132:0:68250:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x10 disposition=0xb, status=0 req@ffff907d90844000 x1874670011439616/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/584 e 0 to 0 dl 1787826342 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826324.732147:0:68250:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b945 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101fa expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.732163:0:68250:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/517 (0 0) 00000080:00200000:0.0:1787826324.732171:0:68250:0:(namei.c:952:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1787826324.732200:0:68250:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x205:0x0], updating i_size 4096 00000080:00200000:0.0:1787826324.732204:0:68250:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x205:0x0](ffff907d93310908): rc = 0 00000080:00010000:0.0:1787826324.732210:0:68250:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x205:0x0](ffff907d93310908) for lock 0xe5f736940357b945 00000080:00010000:0.0:1787826324.732224:0:68250:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x205:0x0](ffff907d93310908) 00000080:00010000:0.0:1787826324.732228:0:68250:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b945 from it ffff9dd142267a28 00010000:00010000:0.0:1787826324.732232:0:68250:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b945 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101fa expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.732241:0:68250:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b945 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101fa expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826324.732280:0:68250:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:0.0:1787826324.732302:0:68250:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), flags=13 00000080:00200000:0.0:1787826324.732308:0:68250:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), intent=getattr 00000002:00010000:0.0:1787826324.732331:0:68250:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x205:0x0], intent: getattr flags 01 00010000:00010000:0.0:1787826324.732381:0:68250:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b94c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.732389:0:68250:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b94c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.732399:0:68250:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b94c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826324.732425:0:68250:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68250:1874670011439744:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826324.732535:0:68250:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd300 going to sleep for 13 seconds 00000100:00100000:0.0:1787826324.744703:0:68250:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68250:1874670011439744:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826324.744748:0:68250:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b94c lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.744767:0:68250:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b94c lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.744775:0:68250:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b94c lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.744783:0:68250:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b94c lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.744792:0:68250:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b94c lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.744799:0:68250:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883b200 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826324.744804:0:68250:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b94c lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.744811:0:68250:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b94c lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.744819:0:68250:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b94c lrc: 5/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.744827:0:68250:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826324.744837:0:68250:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b94c lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.744850:0:68250:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357b94c lrc: 0/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826324.744874:0:68250:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff907d90844000 x1874670011439744/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/584 e 0 to 0 dl 1787826342 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826324.744891:0:68250:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x205:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1787826324.744922:0:68250:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 30 res 8589935620/517 (0 0) 00000080:00200000:0.0:1787826324.745032:0:68250:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826324.745052:0:68250:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826324.745064:0:68250:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826324.745077:0:68250:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826324.745084:0:68250:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826324.745091:0:68250:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826324.745096:0:68250:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:0.0:1787826324.745103:0:68250:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142267be8 00010000:00010000:0.0:1787826324.745106:0:68250:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826324.745124:0:68250:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826324.745144:0:68250:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826324.745155:0:68250:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=16408 00000080:00200000:0.0:1787826324.745164:0:68250:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:0.0:1787826324.745179:0:68250:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), flags=16392 00000080:00200000:0.0:1787826324.745184:0:68250:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), intent=getattr 00000002:00010000:0.0:1787826324.745200:0:68250:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x205:0x0], intent: getattr flags 01 00010000:00010000:0.0:1787826324.745250:0:68250:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b953 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.745257:0:68250:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b953 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.745266:0:68250:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b953 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826324.745292:0:68250:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68250:1874670011439872:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826324.745405:0:68250:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd300 going to sleep for 13 seconds 00000100:00100000:0.0:1787826324.747995:0:68250:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68250:1874670011439872:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826324.748021:0:68250:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b953 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.748037:0:68250:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b953 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.748044:0:68250:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b953 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.748052:0:68250:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b953 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.748062:0:68250:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b953 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.748074:0:68250:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883ac00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826324.748080:0:68250:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b953 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.748087:0:68250:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b953 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.748096:0:68250:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b953 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.748105:0:68250:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826324.748125:0:68250:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b953 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.748134:0:68250:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b953 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826324.748156:0:68250:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff907d90844000 x1874670011439872/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/584 e 0 to 0 dl 1787826342 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826324.748172:0:68250:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x205:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1787826324.748201:0:68250:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 30 res 8589935620/517 (0 0) 00000080:00200000:0.0:1787826324.748298:0:68250:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826324.748317:0:68250:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826324.748328:0:68250:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826324.748338:0:68250:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826324.748350:0:68250:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826324.748355:0:68250:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826324.748360:0:68250:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:0.0:1787826324.748366:0:68250:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142267b58 00010000:00010000:0.0:1787826324.748370:0:68250:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826324.748380:0:68250:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1787826324.748395:0:68250:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826324.748404:0:68250:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=13 00000080:00200008:0.0:1787826324.748415:0:68250:0:(file.c:6313:ll_getattr_dentry()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1787826324.748420:0:68250:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908),name=d204d.sanity-flr 00000002:00010000:0.0:1787826324.748434:0:68250:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x205:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826324.748439:0:68250:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 30 res 8589935620/517 (0 0) 00010000:00010000:0.0:1787826324.748479:0:68250:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.748486:0:68250:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.748494:0:68250:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826324.748517:0:68250:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68250:1874670011440000:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826324.748653:0:68250:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd300 going to sleep for 13 seconds 00000100:00100000:0.0:1787826324.753678:0:68250:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68250:1874670011440000:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826324.753714:0:68250:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10232 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.753747:0:68250:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 4/1,0 mode: --/PR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10232 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.753756:0:68250:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 4/1,0 mode: --/PR res: [0x200000404:0x205:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10232 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.753772:0:68250:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10232 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.753786:0:68250:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10232 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.753801:0:68250:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10232 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.753810:0:68250:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10232 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.753819:0:68250:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10232 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826324.753832:0:68250:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d90844000 x1874670011440000/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/584 e 0 to 0 dl 1787826342 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826324.753848:0:68250:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10232 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.753860:0:68250:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/517 (0 0) 00000080:00200000:0.0:1787826324.753868:0:68250:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1787826324.753897:0:68250:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x205:0x0], updating i_size 4096 00000080:00010000:0.0:1787826324.753904:0:68250:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x205:0x0](ffff907d93310908) for lock 0xe5f736940357b95a 00000080:00010000:0.0:1787826324.753916:0:68250:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x205:0x0](ffff907d93310908) 00000080:00010000:0.0:1787826324.753919:0:68250:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b95a from it ffff9dd142267cb8 00010000:00010000:0.0:1787826324.753923:0:68250:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10232 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.753933:0:68250:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10232 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:0.0:1787826324.754649:0:68250:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1787826324.754658:0:68250:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826324.754678:0:68250:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826324.754694:0:68250:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826324.754708:0:68250:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1787826324.754716:0:68250:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826324.754722:0:68250:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826324.754726:0:68250:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:0.0:1787826324.754731:0:68250:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142267cb8 00010000:00010000:0.0:1787826324.754735:0:68250:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826324.754744:0:68250:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1787826324.754888:0:68250:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826324.754897:0:68250:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826324.754904:0:68250:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1787826324.754914:0:68250:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1787826324.754921:0:68250:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826324.754926:0:68250:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826324.754929:0:68250:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:0.0:1787826324.754933:0:68250:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142267b58 00010000:00010000:0.0:1787826324.754936:0:68250:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1787826324.754944:0:68250:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:0.0:1787826324.754954:0:68250:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826324.754961:0:68250:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=24 00000080:00200000:0.0:1787826324.754966:0:68250:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:0.0:1787826324.754977:0:68250:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), flags=8 00000080:00200000:0.0:1787826324.754983:0:68250:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), intent=getattr 00000002:00010000:0.0:1787826324.754999:0:68250:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x205:0x0], intent: getattr flags 01 00010000:00010000:0.0:1787826324.755054:0:68250:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b961 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.755063:0:68250:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b961 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.755075:0:68250:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b961 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826324.755105:0:68250:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68250:1874670011440128:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826324.755249:0:68250:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd300 going to sleep for 13 seconds 00000100:00100000:0.0:1787826324.758959:0:68250:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68250:1874670011440128:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826324.758990:0:68250:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b961 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.759007:0:68250:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b961 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.759015:0:68250:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b961 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.759024:0:68250:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b961 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.759034:0:68250:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b961 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.759039:0:68250:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a200 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826324.759044:0:68250:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b961 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.759052:0:68250:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b961 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.759059:0:68250:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b961 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x20e400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.759066:0:68250:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826324.759075:0:68250:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b961 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.759081:0:68250:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b961 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826324.759102:0:68250:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff907d90844000 x1874670011440128/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/584 e 0 to 0 dl 1787826342 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826324.759132:0:68250:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x205:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1787826324.759165:0:68250:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10232 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826324.759181:0:68250:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10232 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00010000:0.0:1787826324.759192:0:68250:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b95a from it ffff9dd142267930 00010000:00010000:0.0:1787826324.759195:0:68250:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10232 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826324.759204:0:68250:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10232 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826324.759319:0:68250:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826324.759341:0:68250:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826324.759350:0:68250:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1787826324.759358:0:68250:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:0.0:1787826324.759366:0:68250:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826324.759372:0:68250:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826324.759376:0:68250:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:0.0:1787826324.759383:0:68250:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142267bf8 00010000:00010000:0.0:1787826324.759386:0:68250:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1787826324.759394:0:68250:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:0.0:1787826324.759402:0:68250:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826324.759411:0:68250:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=25 00000080:00200000:0.0:1787826324.759418:0:68250:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:0.0:1787826324.759425:0:68250:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204d.sanity-flr, flags=777 00000080:00200000:0.0:1787826324.759429:0:68250:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 3 00000080:00200000:0.0:1787826324.759434:0:68250:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), file ffff907d84ffd380, open_flags a141, mode 81b6 opened 0 00000080:00200000:0.0:1787826324.759446:0:68250:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), intent=open|creat 00000002:00010000:0.0:1787826324.759469:0:68250:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204d.sanity-flr,[0x200000404:0x206:0x0]) in obj [0x200000404:0x205:0x0], intent: open|creat flags 0120502 00010000:00010000:0.0:1787826324.759482:0:68250:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc10232 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826324.759499:0:68250:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b000/0xe5f736940357b945 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x205:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101fa expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826324.759549:0:68250:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 1/0,0 mode: --/PR res: [0x200000404:0x205:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc10232 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826324.759556:0:68250:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826324.759559:0:68250:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b95a lrc: 0/0,0 mode: --/PR res: [0x200000404:0x205:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc10232 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826324.759587:0:68250:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 3/0,1 mode: --/CW res: [0x200000404:0x205:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.759593:0:68250:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 3/0,1 mode: --/CW res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826324.762003:0:68250:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826324.762032:0:68250:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90844000 x1874670011440384/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/67896 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826324.762048:0:68250:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 3/0,1 mode: --/CW res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826324.762088:0:68250:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68250:1874670011440384:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826324.762323:0:68250:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd300 going to sleep for 13 seconds 00000100:00100000:0.0:1787826324.784878:0:68250:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68250:1874670011440384:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826324.784915:0:68250:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 4/0,1 mode: --/CW res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10263 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.784941:0:68250:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 4/0,1 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10263 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.784951:0:68250:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 4/0,1 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10263 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.784964:0:68250:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10263 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.784977:0:68250:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10263 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.784993:0:68250:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10263 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.785002:0:68250:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10263 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.785009:0:68250:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10263 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826324.785022:0:68250:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x8000037, status=0 req@ffff907d90844000 x1874670011440384/t8589942995(8589942995) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826342 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1787826324.785037:0:68250:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90844000 x1874670011440384/t8589942995(8589942995) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826342 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826324.785049:0:68250:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10263 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.785063:0:68250:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/518 (0 0) 00000080:00200000:0.0:1787826324.785070:0:68250:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x200000404:0x205:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1787826324.785085:0:68250:0:(namei.c:1628:ll_create_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), intent=open|creat 00000080:00200000:0.0:1787826324.785130:0:68250:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88) 00000080:00200000:0.0:1787826324.785139:0:68250:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x206:0x0], updating i_size 0 00000080:00200000:0.0:1787826324.785181:0:68250:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x206:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:0.0:1787826324.785186:0:68250:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) 00000080:00010000:0.0:1787826324.785189:0:68250:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) for lock 0xe5f736940357b968 00000080:00010000:0.0:1787826324.785200:0:68250:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) 00000080:00200008:0.0:1787826324.785218:0:68250:0:(file.c:1090:ll_file_open()) START file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88), flags 120501 00000002:00100000:0.0:1787826324.785240:0:68250:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90844000 x1874670011440384/t8589942995(8589942995) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826342 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:0.0:1787826324.785287:0:68250:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b968 from it ffff907d83241c00 00010000:00010000:0.0:1787826324.785292:0:68250:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10263 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.785301:0:68250:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10263 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826324.785329:0:68250:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88), inode mode 81a4 mask 40 00000080:00200000:0.0:1787826324.785447:0:68250:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88), inode mode 81a4 mask 2 00000080:00200000:0.0:1787826324.785460:0:68250:0:(xattr.c:414:ll_xattr_set()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88), xattr lov 00000002:00010000:0.0:1787826324.785483:0:68250:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x206:0x0]) in obj [0x200000404:0x205:0x0], intent: open flags 040000002 00010000:00010000:0.0:1787826324.785548:0:68250:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b96f lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.785561:0:68250:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b96f lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826324.785573:0:68250:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826324.785580:0:68250:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90845680 x1874670011440640/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826324.785590:0:68250:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b96f lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826324.785667:0:68250:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68250:1874670011440640:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826324.785975:0:68250:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd300 going to sleep for 13 seconds 00000100:00100000:2.0:1787826324.793070:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826324.793195:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025501952 00000100:00100000:3.0:1787826324.793207:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025501952 00000100:00100000:3.0:1787826324.793319:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7153 00000100:00100000:3.0:1787826324.793342:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91149a40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32860:x1874670025501952:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826324.793373:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc10263 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826324.793471:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826324.793479:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91149a40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32860:x1874670025501952:12345-192.168.203.104@tcp:104: Request processed in 150us (423us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826324.793487:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7153 00010000:00010000:1.0:1787826324.793561:0:61297:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826324.793570:0:61297:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc10263 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826324.793580:0:61297:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a000 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826324.793586:0:61297:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc10263 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826324.793596:0:61297:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc10263 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826324.793643:0:61297:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 3/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x7918213f4fc10263 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826324.793678:0:61297:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x206:0x0]: losing layout lock 00010000:00010000:1.0:1787826324.793748:0:61297:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 2/0,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc10263 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826324.793756:0:61297:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826324.793781:0:61297:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc10263 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826324.793789:0:61297:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a000/0xe5f736940357b968 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc10263 expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826324.793813:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826324.793815:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826324.793816:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826324.793826:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1787826324.793846:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011440768:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826324.805923:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011440768:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826324.807894:0:68250:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68250:1874670011440640:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826324.807923:0:68250:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b96f lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.807939:0:68250:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b96f lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.807947:0:68250:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b96f lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.807955:0:68250:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b96f lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.807965:0:68250:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b96f lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.807972:0:68250:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a600 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826324.807977:0:68250:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b96f lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.807984:0:68250:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b96f lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.807991:0:68250:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b96f lrc: 5/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.807999:0:68250:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826324.808007:0:68250:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b96f lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.808014:0:68250:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b96f lrc: 0/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68250 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826324.808038:0:68250:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x800002b, status=0 req@ffff907d90845680 x1874670011440640/t8589942996(8589942996) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1056 e 0 to 0 dl 1787826342 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1787826324.808053:0:68250:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90845680 x1874670011440640/t8589942996(8589942996) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1056 e 0 to 0 dl 1787826342 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826324.808074:0:68250:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x205:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1787826324.808187:0:68250:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1787826324.808197:0:68250:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1787826324.808262:0:68250:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x206:0x0], updating i_size 0 00000002:00100000:0.0:1787826324.808283:0:68250:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90845680 x1874670011440640/t8589942996(8589942996) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 968/1056 e 0 to 0 dl 1787826342 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1787826324.808330:0:68250:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d90845680 x1874670011440640/t8589942996(8589942996) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 968/1056 e 0 to 0 dl 1787826342 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1787826324.808340:0:68250:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1787826324.808346:0:68250:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90846d00 x1874670011441024/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1787826324.808372:0:68250:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68250:1874670011441024:192.168.203.104@tcp:35:lfs.0 00000100:00100000:0.0:1787826324.808477:0:68250:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd300 going to sleep for 11 seconds 00000100:00100000:0.0:1787826324.810937:0:68250:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68250:1874670011441024:192.168.203.104@tcp:35:lfs.0 00000002:00100000:0.0:1787826324.810976:0:68250:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d90845680 x1874670011440640/t8589942996(8589942996) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 968/1056 e 0 to 0 dl 1787826342 ref 2 fl Complete:RQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:0.0:1787826324.811048:0:68250:0:(file.c:479:ll_file_release()) START file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88), flags 100001 00000002:00100000:0.0:1787826324.811122:0:68250:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d90844000 x1874670011440384/t8589942995(8589942995) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826342 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1787826324.811134:0:68250:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1787826324.811140:0:68250:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90847480 x1874670011441280/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1787826324.811164:0:68250:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68250:1874670011441280:192.168.203.104@tcp:35:lfs.0 00000100:00100000:0.0:1787826324.811262:0:68250:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd300 going to sleep for 11 seconds 00000100:00100000:0.0:1787826324.820177:0:68250:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68250:1874670011441280:192.168.203.104@tcp:35:lfs.0 00000002:00100000:0.0:1787826324.820228:0:68250:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d90844000 x1874670011440384/t8589942995(8589942995) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826342 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826324.853841:0:68251:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826324.853884:0:68251:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826324.853909:0:68251:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826324.853930:0:68251:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1787826324.853939:0:68251:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826324.853950:0:68251:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826324.853955:0:68251:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:1.0:1787826324.853966:0:68251:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142af7bf8 00010000:00010000:1.0:1787826324.853970:0:68251:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826324.853980:0:68251:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1787826324.854000:0:68251:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826324.854011:0:68251:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:1.0:1787826324.854018:0:68251:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:1.0:1787826324.854194:0:68251:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), file ffff907d934f3680, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1787826324.854203:0:68251:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), intent=open 00000002:00010000:1.0:1787826324.854223:0:68251:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x205:0x0], intent: open flags 0304001 00010000:00010000:1.0:1787826324.854291:0:68251:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b976 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826324.854300:0:68251:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b976 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826324.854314:0:68251:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826324.854330:0:68251:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a12c0 x1874670011441536/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826324.854342:0:68251:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b976 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826324.854370:0:68251:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68251:1874670011441536:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826324.859558:0:68251:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d87248c00 going to sleep for 13 seconds 00000100:00100000:1.0:1787826324.863236:0:68251:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68251:1874670011441536:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826324.863270:0:68251:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b976 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826324.863288:0:68251:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b976 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826324.863296:0:68251:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b976 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826324.863304:0:68251:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b976 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826324.863314:0:68251:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b976 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826324.863320:0:68251:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0a400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826324.863327:0:68251:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b976 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826324.863334:0:68251:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b976 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826324.863341:0:68251:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b976 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826324.863349:0:68251:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826324.863358:0:68251:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b976 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826324.863365:0:68251:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b976 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826324.863390:0:68251:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908a12c0 x1874670011441536/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826342 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826324.863407:0:68251:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x205:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826324.863599:0:68251:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826324.863653:0:68251:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826324.863675:0:68251:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826324.863690:0:68251:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1787826324.863698:0:68251:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826324.863707:0:68251:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826324.863712:0:68251:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:1.0:1787826324.863720:0:68251:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142af7bf8 00010000:00010000:1.0:1787826324.863724:0:68251:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826324.863733:0:68251:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1787826324.863754:0:68251:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826324.863764:0:68251:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=267 00000080:00200000:2.0:1787826324.864181:0:68251:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 44 00000080:00200008:2.0:1787826324.864196:0:68251:0:(file.c:1090:ll_file_open()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00000002:00010000:2.0:1787826324.864220:0:68251:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x205:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:2.0:1787826324.864289:0:68251:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1e00/0xe5f736940357b97d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826324.864298:0:68251:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1e00/0xe5f736940357b97d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826324.864310:0:68251:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826324.864320:0:68251:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d91148000 x1874670011441792/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826324.864329:0:68251:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1e00/0xe5f736940357b97d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826324.864354:0:68251:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68251:1874670011441792:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826324.864641:0:68251:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88d28d80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826324.868591:0:68251:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68251:1874670011441792:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826324.868652:0:68251:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1e00/0xe5f736940357b97d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.868674:0:68251:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1e00/0xe5f736940357b97d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.868684:0:68251:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1e00/0xe5f736940357b97d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.868693:0:68251:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1e00/0xe5f736940357b97d lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.868703:0:68251:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1e00/0xe5f736940357b97d lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.868711:0:68251:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d863c1e00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826324.868717:0:68251:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1e00/0xe5f736940357b97d lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.868801:0:68251:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1e00/0xe5f736940357b97d lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.868810:0:68251:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1e00/0xe5f736940357b97d lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.868819:0:68251:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826324.868830:0:68251:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1e00/0xe5f736940357b97d lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826324.868839:0:68251:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1e00/0xe5f736940357b97d lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68251 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826324.868867:0:68251:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffff907d91148000 x1874670011441792/t8589942999(8589942999) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826342 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1787826324.868883:0:68251:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d91148000 x1874670011441792/t8589942999(8589942999) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826342 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826324.868899:0:68251:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1787826324.868929:0:68251:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x205:0x0], updating i_size 4096 00000002:00100000:0.0:1787826324.868957:0:68251:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d91148000 x1874670011441792/t8589942999(8589942999) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826342 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:0.0:1787826324.869105:0:68251:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908) cmd=c0086915 arg=1c7a3d0 00000100:00100000:0.0:1787826324.869208:0:68251:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68251:1874670011441920:192.168.203.104@tcp:34:lfs.0 00000100:00100000:0.0:1787826324.869335:0:68251:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd300 going to sleep for 13 seconds 00000100:00100000:0.0:1787826324.884924:0:68251:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68251:1874670011441920:192.168.203.104@tcp:34:lfs.0 00000080:00200008:2.0:1787826324.885273:0:68251:0:(file.c:479:ll_file_release()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00000002:00100000:2.0:1787826324.885361:0:68251:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d91148000 x1874670011441792/t8589942999(8589942999) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826342 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1787826324.885374:0:68251:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1787826324.885380:0:68251:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d911492c0 x1874670011442176/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1787826324.885415:0:68251:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68251:1874670011442176:192.168.203.104@tcp:35:lfs.0 00000100:00100000:2.0:1787826324.885543:0:68251:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88d28240 going to sleep for 11 seconds 00000100:00100000:0.0:1787826324.892686:0:68251:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68251:1874670011442176:192.168.203.104@tcp:35:lfs.0 00000002:00100000:0.0:1787826324.892743:0:68251:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d91148000 x1874670011441792/t8589942999(8589942999) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826342 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1787826324.943050:0:68255:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1787826324.943065:0:68255:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826324.943095:0:68255:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826324.943131:0:68255:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1787826324.943151:0:68255:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1787826324.943160:0:68255:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826324.943171:0:68255:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826324.943175:0:68255:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:2.0:1787826324.943184:0:68255:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142ae7cb8 00010000:00010000:2.0:1787826324.943189:0:68255:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1787826324.943200:0:68255:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1787826324.943333:0:68255:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826324.943341:0:68255:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826324.943349:0:68255:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1787826324.943358:0:68255:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1787826324.943366:0:68255:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826324.943370:0:68255:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826324.943373:0:68255:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:2.0:1787826324.943377:0:68255:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142ae7bb0 00010000:00010000:2.0:1787826324.943379:0:68255:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1787826324.943387:0:68255:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:2.0:1787826324.943396:0:68255:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:2.0:1787826324.943404:0:68255:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:2.0:1787826324.943426:0:68255:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffc095908cc 00000080:00200008:2.0:1787826324.943446:0:68255:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:2.0:1787826324.943474:0:68255:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826324.943482:0:68255:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826324.943501:0:68255:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1787826324.943511:0:68255:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:2.0:1787826324.943518:0:68255:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826324.943523:0:68255:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826324.943526:0:68255:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:2.0:1787826324.943530:0:68255:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142ae7bb0 00010000:00010000:2.0:1787826324.943534:0:68255:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1787826324.943542:0:68255:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:2.0:1787826324.943552:0:68255:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:2.0:1787826324.943556:0:68255:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 100000 00000080:00200080:2.0:1787826324.943565:0:68255:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=8080667f arg=7ffc0958f5a0 00000080:00200008:2.0:1787826324.951689:0:68255:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 100000 00000080:00200008:2.0:1787826324.952358:0:68255:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1787826324.952367:0:68255:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826324.952392:0:68255:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826324.952415:0:68255:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1787826324.952434:0:68255:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:2.0:1787826324.952444:0:68255:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826324.952453:0:68255:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826324.952457:0:68255:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:2.0:1787826324.952466:0:68255:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142ae7cb8 00010000:00010000:2.0:1787826324.952470:0:68255:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1787826324.952480:0:68255:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1787826325.376587:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826326.390693:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826327.413021:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826328.309031:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011442432:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826328.309267:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011442304:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826328.309355:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011442688:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826328.310355:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011442560:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826328.314747:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011442432:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826328.316413:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011442688:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826328.317379:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011442560:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826328.322053:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011442304:192.168.203.104@tcp:400:kworker.0 00010000:00010000:2.0:1787826328.437862:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826329.461065:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826330.496981:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826331.510029:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826332.533148:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826333.430243:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011442944:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826333.431657:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011443200:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826333.434986:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011442816:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826333.435093:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011443072:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826333.438637:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011442816:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826333.438658:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011443072:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826333.439561:0:2348:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589943000 gen 1 00000100:00100000:1.0:1787826333.439683:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011442944:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826333.440598:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011443200:192.168.203.104@tcp:400:kworker.0 00010000:00010000:1.0:1787826333.559770:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826334.581245:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826335.604932:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826336.629937:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1787826337.069928:0:68279:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826337.069958:0:68279:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826337.069978:0:68279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:0.0:1787826337.069995:0:68279:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:0.0:1787826337.070004:0:68279:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826337.070011:0:68279:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826337.070015:0:68279:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:0.0:1787826337.070022:0:68279:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142b4fbf8 00010000:00010000:0.0:1787826337.071900:0:68279:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:0.0:1787826337.071921:0:68279:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:0.0:1787826337.071941:0:68279:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826337.071952:0:68279:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=25 00000080:00200000:0.0:1787826337.071959:0:68279:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:0.0:1787826337.071977:0:68279:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 3 00000080:00200000:0.0:1787826337.071982:0:68279:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), file ffff907d91739c80, open_flags 8042, mode 81b6 opened 0 00000080:00200000:0.0:1787826337.071991:0:68279:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), intent=open|creat 00000002:00010000:0.0:1787826337.072016:0:68279:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204d.sanity-flr,[0x200000404:0x207:0x0]) in obj [0x200000404:0x205:0x0], intent: open|creat flags 0100103 00010000:00010000:0.0:1787826337.072086:0:68279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b984 lrc: 3/0,1 mode: --/CW res: [0x200000404:0x205:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.072095:0:68279:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b984 lrc: 3/0,1 mode: --/CW res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68279 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826337.072108:0:68279:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826337.072121:0:68279:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90846580 x1874670011443456/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/67896 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826337.072132:0:68279:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b984 lrc: 3/0,1 mode: --/CW res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68279 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826337.072153:0:68279:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:68279:1874670011443456:192.168.203.104@tcp:101:dd.0 00000100:00100000:0.0:1787826337.072455:0:68279:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd300 going to sleep for 13 seconds 00000100:00100000:0.0:1787826337.075419:0:68279:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:68279:1874670011443456:192.168.203.104@tcp:101:dd.0 00010000:00010000:0.0:1787826337.075450:0:68279:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b984 lrc: 4/0,1 mode: --/CW res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102d3 expref: -99 pid: 68279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.075470:0:68279:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b984 lrc: 4/0,1 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102d3 expref: -99 pid: 68279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.075490:0:68279:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b984 lrc: 4/0,1 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102d3 expref: -99 pid: 68279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.075503:0:68279:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b984 lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc102d3 expref: -99 pid: 68279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.075519:0:68279:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b984 lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102d3 expref: -99 pid: 68279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.075532:0:68279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b984 lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc102d3 expref: -99 pid: 68279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.075540:0:68279:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b984 lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc102d3 expref: -99 pid: 68279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.075547:0:68279:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b984 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc102d3 expref: -99 pid: 68279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826337.075558:0:68279:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x2b, status=0 req@ffff907d90846580 x1874670011443456/t8589943001(8589943001) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/1056 e 0 to 0 dl 1787826355 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1787826337.075571:0:68279:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90846580 x1874670011443456/t8589943001(8589943001) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/1056 e 0 to 0 dl 1787826355 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826337.075582:0:68279:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 448 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b984 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102d3 expref: -99 pid: 68279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1787826337.075967:0:68279:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b984 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102d3 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.075979:0:68279:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/518 (0 0) 00000080:00200000:0.0:1787826337.075986:0:68279:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x200000404:0x205:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1787826337.076086:0:68279:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x206:0x0](ffff907d93315b88): rc = 0 00000080:00200000:0.0:1787826337.076103:0:68279:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x206:0x0]: layout version change: 4294967294 -> 4 00000080:00010000:0.0:1787826337.076109:0:68279:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) for lock 0xe5f736940357b984 00000080:00010000:0.0:1787826337.076122:0:68279:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) 00000080:00200008:0.0:1787826337.076138:0:68279:0:(file.c:1090:ll_file_open()) START file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88), flags 100102 00000002:00100000:0.0:1787826337.076159:0:68279:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90846580 x1874670011443456/t8589943001(8589943001) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 984/1056 e 0 to 0 dl 1787826355 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000080:00010000:0.0:1787826337.076190:0:68279:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b984 from it ffff907d83241120 00010000:00010000:0.0:1787826337.076195:0:68279:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b984 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc102d3 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.076205:0:68279:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b984 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc102d3 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826337.076226:0:68279:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88), inode mode 81a4 mask 46 00000080:00200080:0.0:1787826337.076279:0:68279:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88) cmd=80306d02 arg=7ffdfa93e6f0 00020000:00000080:0.0:1787826337.076289:0:68279:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff907d986d0000: cmd=80306d02 len=0 karg=0000000000000000 uarg=00007ffdfa93e6f0 00000020:00000080:0.0:1787826337.076293:0:68279:0:(lov_obd.c:981:lov_iocontrol()) lustre-clilov-ffff907d986d0000: iocontrol from 'dd' cmd=80306d02 _IOR('m', 2, 48) unknown: rc = -25 00000080:00200008:0.0:1787826337.076299:0:68279:0:(file.c:5224:ll_file_seek()) START file f204d.sanity-flr:[0x200000404:0x206:0x0], offset: 69206016, type: SEEK_CUR 00000080:00200008:0.0:1787826337.076304:0:68279:0:(file.c:5262:ll_file_seek()) COMPLETED file f204d.sanity-flr:[0x200000404:0x206:0x0], offset: 69206016, type: SEEK_CUR, rc = 69206016 00000080:00200008:0.0:1787826337.078013:0:68279:0:(file.c:2652:do_file_write_iter()) START file f204d.sanity-flr:[0x200000404:0x206:0x0], ppos: 69206016, count: 1048576 00000080:00200000:0.0:1787826337.078042:0:68279:0:(file.c:2090:ll_file_io_generic()) f204d.sanity-flr: write ppos: 69206016, bytes: 1048576 00000080:00200000:0.0:1787826337.078048:0:68279:0:(file.c:1848:ll_io_set_mirror()) f204d.sanity-flr: desiginated mirror: 0 00000020:00200000:0.0:1787826337.078056:0:68279:0:(cl_io.c:218:cl_io_rw_init()) header@000000008ae07a1f[0x110, 1, [0x200000404:0x206:0x0] hash] 00000020:00200000:0.0:1787826337.078059:0:68279:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [69206016, 70254592) 0 0 00000080:00200000:0.0:1787826337.078064:0:68279:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826337.078079:0:68279:0:(lov_io.c:571:lov_io_mirror_init()) [0x200000404:0x206:0x0] need write intent for [69206016, 70254592) 00000080:00200000:0.0:1787826337.078085:0:68279:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 4 need write layout 1, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1787826337.078088:0:68279:0:(vvp_io.c:371:vvp_io_fini()) [0x200000404:0x206:0x0] write layout, type 2 [0x4200000, 0x4300000) 00000080:00010000:0.0:1787826337.078101:0:68279:0:(file.c:7076:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000404:0x206:0x0](ffff907d93315b88) 00000002:00010000:0.0:1787826337.078112:0:68279:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x206:0x0]) in obj [0x200000404:0x206:0x0], intent: layout flags 02 00010000:00010000:0.0:1787826337.078133:0:68279:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b984 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc102d3 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826337.078151:0:68279:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x206:0x0]: losing layout lock 00010000:00010000:0.0:1787826337.078194:0:68279:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b984 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc102d3 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826337.078202:0:68279:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826337.078205:0:68279:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357b984 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc102d3 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826337.078237:0:68279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 3/0,1 mode: --/EX res: [0x200000404:0x206:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 68279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.078245:0:68279:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 3/0,1 mode: --/EX res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68279 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826337.078255:0:68279:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826337.078266:0:68279:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90847840 x1874670011443712/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/2048 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826337.078275:0:68279:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 3/0,1 mode: --/EX res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68279 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826337.078293:0:68279:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:68279:1874670011443712:192.168.203.104@tcp:101:dd.0 00000100:00100000:0.0:1787826337.078386:0:68279:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd300 going to sleep for 13 seconds 00000100:00100000:0.0:1787826337.116915:0:68279:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589943002 gen 1 00000100:00100000:0.0:1787826337.116946:0:68279:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d90846580 x1874670011443456/t8589943001(8589943001) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 984/1056 e 0 to 0 dl 1787826355 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1787826337.116962:0:68279:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:68279:1874670011443712:192.168.203.104@tcp:101:dd.0 00010000:00010000:0.0:1787826337.116984:0:68279:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 4/0,1 mode: --/EX res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.117006:0:68279:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 4/0,1 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.117019:0:68279:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 4/0,1 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.117038:0:68279:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 4/0,1 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.117051:0:68279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 5/1,1 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.117058:0:68279:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 5/1,1 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.117065:0:68279:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826337.117073:0:68279:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x400 disposition=0x1, status=0 req@ffff907d90847840 x1874670011443712/t8589943002(8589943002) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/896 e 0 to 0 dl 1787826355 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826337.117090:0:68279:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1787826337.117104:0:68279:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.117115:0:68279:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/518 (0 0) 00000080:00200000:0.0:1787826337.117121:0:68279:0:(namei.c:952:ll_intent_lock()) intent lock 1024 on i1 [0x200000404:0x206:0x0] suppgids 0 0: rc 0 00000080:00010000:0.0:1787826337.117134:0:68279:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) for lock 0xe5f736940357b98b 00000080:00010000:0.0:1787826337.117142:0:68279:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b98b from it ffff9dd142b4fbb8 00010000:00010000:0.0:1787826337.117146:0:68279:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.117153:0:68279:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826337.117168:0:68279:0:(file.c:2240:ll_file_io_generic()) f204d.sanity-flr: 2 io complete with rc: 0, result: 0, restart: 1 00000080:00200000:0.0:1787826337.117172:0:68279:0:(file.c:2247:ll_file_io_generic()) f204d.sanity-flr: restart write from ppos=69206016 bytes=1048576 retries=999 ret=0: rc = 0 00000080:00200000:0.0:1787826337.117178:0:68279:0:(file.c:1848:ll_io_set_mirror()) f204d.sanity-flr: desiginated mirror: 0 00000020:00200000:0.0:1787826337.117186:0:68279:0:(cl_io.c:218:cl_io_rw_init()) header@000000008ae07a1f[0x110, 1, [0x200000404:0x206:0x0] hash] 00000020:00200000:0.0:1787826337.117188:0:68279:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [69206016, 70254592) 0 0 00000080:00200000:0.0:1787826337.117194:0:68279:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 4 restore needed 0 00010000:00010000:0.0:1787826337.117209:0:68279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826337.117219:0:68279:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:0.0:1787826337.117227:0:68279:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x206:0x0](00000000407d6ff3) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826337.117269:0:68279:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826337.117276:0:68279:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826337.117287:0:68279:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826337.117295:0:68279:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826337.117298:0:68279:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826337.117303:0:68279:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826337.117347:0:68279:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x40 00000080:00200000:0.0:1787826337.117364:0:68279:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1787826337.117382:0:68279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826337.117389:0:68279:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1787826337.117396:0:68279:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x206:0x0](00000000407d6ff3) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826337.117410:0:68279:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826337.117414:0:68279:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826337.117416:0:68279:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826337.117418:0:68279:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826337.117431:0:68279:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x40 00000080:00200000:0.0:1787826337.117436:0:68279:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1787826337.117495:0:68279:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x206:0x0]: layout version change: 4294967294 -> 6 00010000:00010000:0.0:1787826337.117501:0:68279:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826337.117508:0:68279:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00020000:00200000:0.0:1787826337.117520:0:68279:0:(lov_io.c:311:lov_io_mirror_write_intent()) [0x200000404:0x206:0x0]there are 0 components to be staled to modify file extent [0x4000000, 0x8000000), iot: 2 00020000:00200000:0.0:1787826337.117524:0:68279:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826337.117526:0:68279:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00000080:00200000:0.0:1787826337.117530:0:68279:0:(file.c:2170:ll_file_io_generic()) Range lock [16896, 17151] 00020000:00200000:0.0:1787826337.117536:0:68279:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 69206016 chunk: [69206016, 70254592) 70254592, 1048576 00020000:00200000:0.0:1787826337.117538:0:68279:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00000080:00200000:0.0:1787826337.117549:0:68279:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [16896, 17151] 00000008:00010000:0.0:1787826337.117559:0:68279:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1787826337.117564:0:68279:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da032e9b0, osc lock ffff907d96368618, flags 0x0 00010000:00010000:0.0:1787826337.117600:0:68279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.117654:0:68279:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [69206016->70254591] (req 69206016->70254591) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.117663:0:68279:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [69206016->70254591] (req 69206016->70254591) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826337.117681:0:68279:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:68279:1874670011443840:192.168.203.104@tcp:101:dd.0 00000100:00100000:0.0:1787826337.117760:0:68279:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd300 going to sleep for 11 seconds 00000100:00100000:0.0:1787826337.123452:0:68279:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:68279:1874670011443840:192.168.203.104@tcp:101:dd.0 00010000:00010000:0.0:1787826337.123479:0:68279:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.123498:0:68279:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.123510:0:68279:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.123524:0:68279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.123537:0:68279:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.123550:0:68279:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.123558:0:68279:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826337.123584:0:68279:0:(vvp_io.c:1339:vvp_io_write_start()) f204d.sanity-flr: write [69206016, 70254592) append 0 append lockpos 0 00000002:00010000:0.0:1787826337.123729:0:68279:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x0:0x0:0x0]) in obj [0x200000404:0x206:0x0], intent: getxattr flags 0 00010000:00010000:0.0:1787826337.123774:0:68279:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b999 lrc: 3/1,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.123786:0:68279:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b999 lrc: 3/1,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68279 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.123795:0:68279:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b999 lrc: 3/1,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68279 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826337.123814:0:68279:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:68279:1874670011443968:192.168.203.104@tcp:101:dd.0 00000100:00100000:0.0:1787826337.123886:0:68279:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd300 going to sleep for 13 seconds 00000100:00100000:0.0:1787826337.126058:0:68279:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:68279:1874670011443968:192.168.203.104@tcp:101:dd.0 00010000:00010000:0.0:1787826337.126080:0:68279:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b999 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102fd expref: -99 pid: 68279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.126094:0:68279:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b999 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc102fd expref: -99 pid: 68279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.126105:0:68279:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b999 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102fd expref: -99 pid: 68279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826337.126117:0:68279:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x80 disposition=0x0, status=0 req@ffff907d90847840 x1874670011443968/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 568/1648 e 0 to 0 dl 1787826355 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826337.126192:0:68279:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b999 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102fd expref: -99 pid: 68279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.126205:0:68279:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/518 (0 0) 00000080:00200000:0.0:1787826337.126211:0:68279:0:(namei.c:952:ll_intent_lock()) intent lock 128 on i1 [0x200000404:0x206:0x0] suppgids 0 -1: rc 0 00000080:00010000:0.0:1787826337.126233:0:68279:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) for lock 0xe5f736940357b999 00000080:00010000:0.0:1787826337.126242:0:68279:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b999 from it ffff9dd142b4f8e8 00010000:00010000:0.0:1787826337.126247:0:68279:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b999 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc102fd expref: -99 pid: 68279 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.126256:0:68279:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b999 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc102fd expref: -99 pid: 68279 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826337.128246:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16896 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.128310:0:68279:0:(rw26.c:875:ll_write_end()) pos 69206016, len 4096, copied 4096 00000080:00200000:0.0:1787826337.128320:0:68279:0:(rw26.c:933:ll_write_end()) page@000000001a993665[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16896] 00000080:00200000:0.0:1787826337.128323:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000cb9f8014 17ffffc0001001 3:0 ffff907d88f89488 16896 lru 00000080:00200000:0.0:1787826337.128332:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000d867992a 16896: 1< 0 - - > 2< 69206016 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.128334:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000001a993665 00000080:00200000:0.0:1787826337.128336:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1787826337.128339:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16897 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.128356:0:68279:0:(rw26.c:875:ll_write_end()) pos 69210112, len 4096, copied 4096 00000080:00200000:0.0:1787826337.128361:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000792235b3[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16897] 00000080:00200000:0.0:1787826337.128363:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000aea91d6a 17ffffc0001001 3:0 ffff907d88f893a0 16897 lru 00000080:00200000:0.0:1787826337.128369:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000085fe9a64 16897: 1< 0 - - > 2< 69210112 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.128371:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000792235b3 00000080:00200000:0.0:1787826337.128373:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 2. 00000080:00200000:0.0:1787826337.128375:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16898 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.128391:0:68279:0:(rw26.c:875:ll_write_end()) pos 69214208, len 4096, copied 4096 00000080:00200000:0.0:1787826337.128395:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000f25a8968[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16898] 00000080:00200000:0.0:1787826337.128399:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000000ad8e4f8 17ffffc0001001 3:0 ffff907d88f89e80 16898 lru 00000080:00200000:0.0:1787826337.128404:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000697ceae3 16898: 1< 0 - - > 2< 69214208 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.128406:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000f25a8968 00000080:00200000:0.0:1787826337.128407:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 3. 00000080:00200000:0.0:1787826337.128409:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16899 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.128433:0:68279:0:(rw26.c:875:ll_write_end()) pos 69218304, len 4096, copied 4096 00000080:00200000:0.0:1787826337.128438:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000bb95d188[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16899] 00000080:00200000:0.0:1787826337.128441:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000365b1bd2 17ffffc0001001 3:0 ffff907d88f89ae0 16899 lru 00000080:00200000:0.0:1787826337.128447:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000001dfa6ab4 16899: 1< 0 - - > 2< 69218304 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.128448:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000bb95d188 00000080:00200000:0.0:1787826337.128450:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 4. 00000080:00200000:0.0:1787826337.128452:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16900 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.128465:0:68279:0:(rw26.c:875:ll_write_end()) pos 69222400, len 4096, copied 4096 00000080:00200000:0.0:1787826337.128469:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000e04e2575[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16900] 00000080:00200000:0.0:1787826337.128472:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000b1253e75 17ffffc0001001 3:0 ffff907d88f89000 16900 lru 00000080:00200000:0.0:1787826337.128478:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000007aa0e08f 16900: 1< 0 - - > 2< 69222400 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.128480:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000e04e2575 00000080:00200000:0.0:1787826337.128481:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 5. 00000080:00200000:0.0:1787826337.128482:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16901 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.128507:0:68279:0:(rw26.c:875:ll_write_end()) pos 69226496, len 4096, copied 4096 00000080:00200000:0.0:1787826337.128511:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000ba120f42[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16901] 00000080:00200000:0.0:1787826337.128514:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000a701f6e9 17ffffc0001001 3:0 ffff907d88f892b8 16901 lru 00000080:00200000:0.0:1787826337.128519:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000d376acd3 16901: 1< 0 - - > 2< 69226496 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.128521:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000ba120f42 00000080:00200000:0.0:1787826337.128522:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 6. 00000080:00200000:0.0:1787826337.128524:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16902 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.128538:0:68279:0:(rw26.c:875:ll_write_end()) pos 69230592, len 4096, copied 4096 00000080:00200000:0.0:1787826337.128542:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000b922a6c1[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16902] 00000080:00200000:0.0:1787826337.128545:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000f572b452 17ffffc0001001 3:0 ffff907d88f891d0 16902 lru 00000080:00200000:0.0:1787826337.128550:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000eac160ef 16902: 1< 0 - - > 2< 69230592 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.128552:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000b922a6c1 00000080:00200000:0.0:1787826337.128553:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 7. 00000080:00200000:0.0:1787826337.128555:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16903 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.128566:0:68279:0:(rw26.c:875:ll_write_end()) pos 69234688, len 4096, copied 4096 00000080:00200000:0.0:1787826337.128570:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000b8537ed0[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16903] 00000080:00200000:0.0:1787826337.128573:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000eee890f4 17ffffc0001001 3:0 ffff907d88f89910 16903 lru 00000080:00200000:0.0:1787826337.128578:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000054a41793 16903: 1< 0 - - > 2< 69234688 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.128580:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000b8537ed0 00000080:00200000:0.0:1787826337.128581:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 8. 00000080:00200000:0.0:1787826337.128593:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16904 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.128626:0:68279:0:(rw26.c:875:ll_write_end()) pos 69238784, len 4096, copied 4096 00000080:00200000:0.0:1787826337.128631:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000b676b96a[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16904] 00000080:00200000:0.0:1787826337.128633:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000005045eaeb 17ffffc0001001 3:0 ffff907d88f890e8 16904 lru 00000080:00200000:0.0:1787826337.128639:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000029afc2c3 16904: 1< 0 - - > 2< 69238784 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.128641:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000b676b96a 00000080:00200000:0.0:1787826337.128642:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 9. 00000080:00200000:0.0:1787826337.128644:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16905 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.128668:0:68279:0:(rw26.c:875:ll_write_end()) pos 69242880, len 4096, copied 4096 00000080:00200000:0.0:1787826337.128672:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000dca5499c[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16905] 00000080:00200000:0.0:1787826337.128675:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000004ac8c872 17ffffc0001021 2:0 ffff907d88f89658 16905 lru 00000080:00200000:0.0:1787826337.128680:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000588cad3b 16905: 1< 0 - - > 2< 69242880 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.128682:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000dca5499c 00000080:00200000:0.0:1787826337.128683:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 10. 00000080:00200000:0.0:1787826337.128685:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16906 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.128698:0:68279:0:(rw26.c:875:ll_write_end()) pos 69246976, len 4096, copied 4096 00000080:00200000:0.0:1787826337.128702:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000f5072c8d[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16906] 00000080:00200000:0.0:1787826337.128705:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000c9b1725d 17ffffc0001001 3:0 ffff907d88f89d98 16906 lru 00000080:00200000:0.0:1787826337.128710:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000ca2e413f 16906: 1< 0 - - > 2< 69246976 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.128712:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000f5072c8d 00000080:00200000:0.0:1787826337.128714:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 11. 00000080:00200000:0.0:1787826337.128715:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16907 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.129010:0:68279:0:(rw26.c:875:ll_write_end()) pos 69251072, len 4096, copied 4096 00000080:00200000:0.0:1787826337.129015:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000a9e15924[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16907] 00000080:00200000:0.0:1787826337.129018:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000ad822e25 17ffffc0001001 3:0 ffff907d88f89828 16907 lru 00000080:00200000:0.0:1787826337.129023:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000056f6d9a 16907: 1< 0 - - > 2< 69251072 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.129034:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000a9e15924 00000080:00200000:0.0:1787826337.129035:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 12. 00000080:00200000:0.0:1787826337.129038:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16908 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.129056:0:68279:0:(rw26.c:875:ll_write_end()) pos 69255168, len 4096, copied 4096 00000080:00200000:0.0:1787826337.129060:0:68279:0:(rw26.c:933:ll_write_end()) page@000000007e78a064[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16908] 00000080:00200000:0.0:1787826337.129063:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000752e66a7 17ffffc0001001 3:0 ffff907d88f89570 16908 lru 00000080:00200000:0.0:1787826337.129068:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000001624d4b5 16908: 1< 0 - - > 2< 69255168 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.129071:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000007e78a064 00000080:00200000:0.0:1787826337.129072:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 13. 00000080:00200000:0.0:1787826337.129074:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16909 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.129087:0:68279:0:(rw26.c:875:ll_write_end()) pos 69259264, len 4096, copied 4096 00000080:00200000:0.0:1787826337.129092:0:68279:0:(rw26.c:933:ll_write_end()) page@000000000f826417[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16909] 00000080:00200000:0.0:1787826337.129095:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000009849b308 17ffffc0001001 3:0 ffff907d88f899f8 16909 lru 00000080:00200000:0.0:1787826337.129100:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000002c596059 16909: 1< 0 - - > 2< 69259264 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.129103:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000000f826417 00000080:00200000:0.0:1787826337.129104:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 14. 00000080:00200000:0.0:1787826337.129106:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16910 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.129119:0:68279:0:(rw26.c:875:ll_write_end()) pos 69263360, len 4096, copied 4096 00000080:00200000:0.0:1787826337.129123:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000f1568777[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16910] 00000080:00200000:0.0:1787826337.129126:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000098def80c 17ffffc0001001 3:0 ffff907d88f89cb0 16910 lru 00000080:00200000:0.0:1787826337.129131:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000000fa2b45f 16910: 1< 0 - - > 2< 69263360 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.129134:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000f1568777 00000080:00200000:0.0:1787826337.129135:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 15. 00000080:00200000:0.0:1787826337.129137:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16911 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.129154:0:68279:0:(rw26.c:875:ll_write_end()) pos 69267456, len 4096, copied 4096 00000080:00200000:0.0:1787826337.129158:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000069eb338a[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16911] 00000080:00200000:0.0:1787826337.129161:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000b0bcb7d3 17ffffc0001001 3:0 ffff907d88f89740 16911 lru 00000080:00200000:0.0:1787826337.129223:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000bdb62e7c 16911: 1< 0 - - > 2< 69267456 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.129226:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000069eb338a 00000080:00200000:0.0:1787826337.129228:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 16. 00000080:00200000:0.0:1787826337.129230:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16912 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.129248:0:68279:0:(rw26.c:875:ll_write_end()) pos 69271552, len 4096, copied 4096 00000080:00200000:0.0:1787826337.129253:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000a07de997[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16912] 00000080:00200000:0.0:1787826337.129256:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000905b810d 17ffffc0001001 3:0 ffff907d88f89bc8 16912 lru 00000080:00200000:0.0:1787826337.129262:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000005be88947 16912: 1< 0 - - > 2< 69271552 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.129264:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000a07de997 00000080:00200000:0.0:1787826337.129265:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 17. 00000080:00200000:0.0:1787826337.129267:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16913 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.129283:0:68279:0:(rw26.c:875:ll_write_end()) pos 69275648, len 4096, copied 4096 00000080:00200000:0.0:1787826337.129288:0:68279:0:(rw26.c:933:ll_write_end()) page@000000002ee33717[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16913] 00000080:00200000:0.0:1787826337.129291:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000d7730875 17ffffc0001001 3:0 ffff907d834bb570 16913 lru 00000080:00200000:0.0:1787826337.129296:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000003ff1ad0f 16913: 1< 0 - - > 2< 69275648 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.129299:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000002ee33717 00000080:00200000:0.0:1787826337.129299:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 18. 00000080:00200000:0.0:1787826337.129301:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16914 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.129315:0:68279:0:(rw26.c:875:ll_write_end()) pos 69279744, len 4096, copied 4096 00000080:00200000:0.0:1787826337.129320:0:68279:0:(rw26.c:933:ll_write_end()) page@000000003fe3062a[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16914] 00000080:00200000:0.0:1787826337.129322:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000e1049808 17ffffc0001001 3:0 ffff907d834bb828 16914 lru 00000080:00200000:0.0:1787826337.129328:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000053d4cb31 16914: 1< 0 - - > 2< 69279744 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.129330:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000003fe3062a 00000080:00200000:0.0:1787826337.129331:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 19. 00000080:00200000:0.0:1787826337.129333:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16915 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.129345:0:68279:0:(rw26.c:875:ll_write_end()) pos 69283840, len 4096, copied 4096 00000080:00200000:0.0:1787826337.129350:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000e7b9622c[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16915] 00000080:00200000:0.0:1787826337.129353:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000ad53d916 17ffffc0001001 3:0 ffff907d834bbd98 16915 lru 00000080:00200000:0.0:1787826337.129358:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000f2a8a314 16915: 1< 0 - - > 2< 69283840 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.129360:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000e7b9622c 00000080:00200000:0.0:1787826337.129361:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 20. 00000080:00200000:0.0:1787826337.129363:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16916 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.129375:0:68279:0:(rw26.c:875:ll_write_end()) pos 69287936, len 4096, copied 4096 00000080:00200000:0.0:1787826337.129379:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000646ebdea[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16916] 00000080:00200000:0.0:1787826337.129381:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000013125239 17ffffc0001001 3:0 ffff907d834bb658 16916 lru 00000080:00200000:0.0:1787826337.129386:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000c7445f21 16916: 1< 0 - - > 2< 69287936 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.129389:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000646ebdea 00000080:00200000:0.0:1787826337.129390:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 21. 00000080:00200000:0.0:1787826337.129391:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16917 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.129404:0:68279:0:(rw26.c:875:ll_write_end()) pos 69292032, len 4096, copied 4096 00000080:00200000:0.0:1787826337.129407:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000d4611838[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16917] 00000080:00200000:0.0:1787826337.129410:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000a2a1a138 17ffffc0001001 3:0 ffff907d834bb0e8 16917 lru 00000080:00200000:0.0:1787826337.129415:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000000b9f74f9 16917: 1< 0 - - > 2< 69292032 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.129416:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000d4611838 00000080:00200000:0.0:1787826337.129417:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 22. 00000080:00200000:0.0:1787826337.129419:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16918 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.129433:0:68279:0:(rw26.c:875:ll_write_end()) pos 69296128, len 4096, copied 4096 00000080:00200000:0.0:1787826337.129437:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000bd4b8a5c[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16918] 00000080:00200000:0.0:1787826337.129439:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000eb47e80c 17ffffc0001001 3:0 ffff907d834bb910 16918 lru 00000080:00200000:0.0:1787826337.129444:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000004a5f708f 16918: 1< 0 - - > 2< 69296128 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.129445:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000bd4b8a5c 00000080:00200000:0.0:1787826337.129446:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 23. 00000080:00200000:0.0:1787826337.129448:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16919 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.129462:0:68279:0:(rw26.c:875:ll_write_end()) pos 69300224, len 4096, copied 4096 00000080:00200000:0.0:1787826337.129467:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000023277c7d[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16919] 00000080:00200000:0.0:1787826337.129469:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000005e46601f 17ffffc0001001 3:0 ffff907d834bb1d0 16919 lru 00000080:00200000:0.0:1787826337.129473:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000005d26ebff 16919: 1< 0 - - > 2< 69300224 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.129475:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000023277c7d 00000080:00200000:0.0:1787826337.129476:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 24. 00000080:00200000:0.0:1787826337.129478:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16920 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.129494:0:68279:0:(rw26.c:875:ll_write_end()) pos 69304320, len 4096, copied 4096 00000080:00200000:0.0:1787826337.129499:0:68279:0:(rw26.c:933:ll_write_end()) page@000000006d309b58[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16920] 00000080:00200000:0.0:1787826337.129502:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000ed54d56c 17ffffc0001021 2:0 ffff907d834bb2b8 16920 lru 00000080:00200000:0.0:1787826337.129507:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000b0afdc8e 16920: 1< 0 - - > 2< 69304320 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.129510:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000006d309b58 00000080:00200000:0.0:1787826337.129511:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 25. 00000080:00200000:0.0:1787826337.129513:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16921 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.129526:0:68279:0:(rw26.c:875:ll_write_end()) pos 69308416, len 4096, copied 4096 00000080:00200000:0.0:1787826337.129529:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000056a1f7cd[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16921] 00000080:00200000:0.0:1787826337.129532:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000ea09475a 17ffffc0001001 3:0 ffff907d834bb000 16921 lru 00000080:00200000:0.0:1787826337.129536:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000006443bc39 16921: 1< 0 - - > 2< 69308416 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.129538:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000056a1f7cd 00000080:00200000:0.0:1787826337.129539:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 26. 00000080:00200000:0.0:1787826337.129540:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16922 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.129552:0:68279:0:(rw26.c:875:ll_write_end()) pos 69312512, len 4096, copied 4096 00000080:00200000:0.0:1787826337.129556:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000a6bc2fce[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16922] 00000080:00200000:0.0:1787826337.129559:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000b7af2fe1 17ffffc0001001 3:0 ffff907d834bbae0 16922 lru 00000080:00200000:0.0:1787826337.129563:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000e30ccf0c 16922: 1< 0 - - > 2< 69312512 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.129565:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000a6bc2fce 00000080:00200000:0.0:1787826337.129566:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 27. 00000080:00200000:0.0:1787826337.129568:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16923 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.129579:0:68279:0:(rw26.c:875:ll_write_end()) pos 69316608, len 4096, copied 4096 00000080:00200000:0.0:1787826337.130997:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000006339228[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16923] 00000080:00200000:0.0:1787826337.131005:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000823f4cad 17ffffc0001001 3:0 ffff907d834bbe80 16923 lru 00000080:00200000:0.0:1787826337.131013:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000a5289019 16923: 1< 0 - - > 2< 69316608 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.131016:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000006339228 00000080:00200000:0.0:1787826337.131018:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 28. 00000080:00200000:0.0:1787826337.131023:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16924 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.131068:0:68279:0:(rw26.c:875:ll_write_end()) pos 69320704, len 4096, copied 4096 00000080:00200000:0.0:1787826337.131074:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000b6381bca[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16924] 00000080:00200000:0.0:1787826337.131077:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000e010aec8 17ffffc0001001 3:0 ffff907d834bb3a0 16924 lru 00000080:00200000:0.0:1787826337.131082:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000d85e9b80 16924: 1< 0 - - > 2< 69320704 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.131084:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000b6381bca 00000080:00200000:0.0:1787826337.131085:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 29. 00000080:00200000:0.0:1787826337.131087:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16925 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.131101:0:68279:0:(rw26.c:875:ll_write_end()) pos 69324800, len 4096, copied 4096 00000080:00200000:0.0:1787826337.131105:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000047f263cd[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16925] 00000080:00200000:0.0:1787826337.131108:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000009800d303 17ffffc0001001 3:0 ffff907d834bb488 16925 lru 00000080:00200000:0.0:1787826337.131113:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000247533e7 16925: 1< 0 - - > 2< 69324800 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.131115:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000047f263cd 00000080:00200000:0.0:1787826337.131116:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 30. 00000080:00200000:0.0:1787826337.131118:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16926 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.131134:0:68279:0:(rw26.c:875:ll_write_end()) pos 69328896, len 4096, copied 4096 00000080:00200000:0.0:1787826337.131138:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000078debf1a[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16926] 00000080:00200000:0.0:1787826337.131140:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000696083c0 17ffffc0001001 3:0 ffff907d834bb740 16926 lru 00000080:00200000:0.0:1787826337.131145:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000a5223a53 16926: 1< 0 - - > 2< 69328896 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.131147:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000078debf1a 00000080:00200000:0.0:1787826337.131148:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 31. 00000080:00200000:0.0:1787826337.131150:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16927 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.131163:0:68279:0:(rw26.c:875:ll_write_end()) pos 69332992, len 4096, copied 4096 00000080:00200000:0.0:1787826337.131167:0:68279:0:(rw26.c:933:ll_write_end()) page@000000004288afc5[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16927] 00000080:00200000:0.0:1787826337.131170:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000036a53140 17ffffc0001001 3:0 ffff907d834bbbc8 16927 lru 00000080:00200000:0.0:1787826337.131174:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000e1cfe524 16927: 1< 0 - - > 2< 69332992 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.131186:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000004288afc5 00000080:00200000:0.0:1787826337.131187:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 32. 00000080:00200000:0.0:1787826337.131189:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16928 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.131203:0:68279:0:(rw26.c:875:ll_write_end()) pos 69337088, len 4096, copied 4096 00000080:00200000:0.0:1787826337.131207:0:68279:0:(rw26.c:933:ll_write_end()) page@000000004b6dc4b3[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16928] 00000080:00200000:0.0:1787826337.131210:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000502c8516 17ffffc0001001 3:0 ffff907d834bbcb0 16928 lru 00000080:00200000:0.0:1787826337.131215:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000d6f246f8 16928: 1< 0 - - > 2< 69337088 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.131217:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000004b6dc4b3 00000080:00200000:0.0:1787826337.131218:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 33. 00000080:00200000:0.0:1787826337.131220:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16929 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.131234:0:68279:0:(rw26.c:875:ll_write_end()) pos 69341184, len 4096, copied 4096 00000080:00200000:0.0:1787826337.131239:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000c6948333[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16929] 00000080:00200000:0.0:1787826337.131242:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000004a0f5ed9 17ffffc0001001 3:0 ffff907d834bb9f8 16929 lru 00000080:00200000:0.0:1787826337.131248:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000f323d82e 16929: 1< 0 - - > 2< 69341184 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.131250:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000c6948333 00000080:00200000:0.0:1787826337.131251:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 34. 00000080:00200000:0.0:1787826337.131254:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16930 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.131268:0:68279:0:(rw26.c:875:ll_write_end()) pos 69345280, len 4096, copied 4096 00000080:00200000:0.0:1787826337.131272:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000f608758c[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16930] 00000080:00200000:0.0:1787826337.131274:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000092a5fcd9 17ffffc0001001 3:0 ffff907d916542b8 16930 lru 00000080:00200000:0.0:1787826337.131279:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000014ace7d7 16930: 1< 0 - - > 2< 69345280 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.131281:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000f608758c 00000080:00200000:0.0:1787826337.131282:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 35. 00000080:00200000:0.0:1787826337.131284:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16931 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.131596:0:68279:0:(rw26.c:875:ll_write_end()) pos 69349376, len 4096, copied 4096 00000080:00200000:0.0:1787826337.131602:0:68279:0:(rw26.c:933:ll_write_end()) page@000000001cec0928[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16931] 00000080:00200000:0.0:1787826337.131604:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000d749c429 17ffffc0001001 3:0 ffff907d91654000 16931 lru 00000080:00200000:0.0:1787826337.131675:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000dc4c880f 16931: 1< 0 - - > 2< 69349376 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.131677:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000001cec0928 00000080:00200000:0.0:1787826337.131678:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 36. 00000080:00200000:0.0:1787826337.131680:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16932 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.131696:0:68279:0:(rw26.c:875:ll_write_end()) pos 69353472, len 4096, copied 4096 00000080:00200000:0.0:1787826337.131700:0:68279:0:(rw26.c:933:ll_write_end()) page@000000001626f632[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16932] 00000080:00200000:0.0:1787826337.131703:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000648197c8 17ffffc0001001 3:0 ffff907d91654ae0 16932 lru 00000080:00200000:0.0:1787826337.131707:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000007a6f586a 16932: 1< 0 - - > 2< 69353472 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.131709:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000001626f632 00000080:00200000:0.0:1787826337.131710:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 37. 00000080:00200000:0.0:1787826337.131711:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16933 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.131726:0:68279:0:(rw26.c:875:ll_write_end()) pos 69357568, len 4096, copied 4096 00000080:00200000:0.0:1787826337.131730:0:68279:0:(rw26.c:933:ll_write_end()) page@000000004cbc31b9[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16933] 00000080:00200000:0.0:1787826337.131733:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000dc9ada7d 17ffffc0001001 3:0 ffff907d91654e80 16933 lru 00000080:00200000:0.0:1787826337.131737:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000f7b7d26f 16933: 1< 0 - - > 2< 69357568 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.131739:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000004cbc31b9 00000080:00200000:0.0:1787826337.131740:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 38. 00000080:00200000:0.0:1787826337.131742:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16934 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.131755:0:68279:0:(rw26.c:875:ll_write_end()) pos 69361664, len 4096, copied 4096 00000080:00200000:0.0:1787826337.131759:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000af7fb27e[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16934] 00000080:00200000:0.0:1787826337.131762:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000005f6e0954 17ffffc0001001 3:0 ffff907d916543a0 16934 lru 00000080:00200000:0.0:1787826337.131768:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000001b5ab9ea 16934: 1< 0 - - > 2< 69361664 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.131770:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000af7fb27e 00000080:00200000:0.0:1787826337.131771:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 39. 00000080:00200000:0.0:1787826337.131773:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16935 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.131792:0:68279:0:(rw26.c:875:ll_write_end()) pos 69365760, len 4096, copied 4096 00000080:00200000:0.0:1787826337.131796:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000002403ca1[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16935] 00000080:00200000:0.0:1787826337.131799:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000009e6926cc 17ffffc0001021 2:0 ffff907d91654488 16935 lru 00000080:00200000:0.0:1787826337.131803:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000bbddf62d 16935: 1< 0 - - > 2< 69365760 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.131806:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000002403ca1 00000080:00200000:0.0:1787826337.131807:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 40. 00000080:00200000:0.0:1787826337.131808:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16936 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.131821:0:68279:0:(rw26.c:875:ll_write_end()) pos 69369856, len 4096, copied 4096 00000080:00200000:0.0:1787826337.131825:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000542ee64c[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16936] 00000080:00200000:0.0:1787826337.131828:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000003fd06255 17ffffc0001001 3:0 ffff907d91654740 16936 lru 00000080:00200000:0.0:1787826337.131833:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000085d6533d 16936: 1< 0 - - > 2< 69369856 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.131835:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000542ee64c 00000080:00200000:0.0:1787826337.131835:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 41. 00000080:00200000:0.0:1787826337.131837:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16937 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.131849:0:68279:0:(rw26.c:875:ll_write_end()) pos 69373952, len 4096, copied 4096 00000080:00200000:0.0:1787826337.131853:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000aa2bec03[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16937] 00000080:00200000:0.0:1787826337.131855:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000005760e0 17ffffc0001001 3:0 ffff907d91654bc8 16937 lru 00000080:00200000:0.0:1787826337.131860:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000087a77ad1 16937: 1< 0 - - > 2< 69373952 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.131862:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000aa2bec03 00000080:00200000:0.0:1787826337.131863:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 42. 00000080:00200000:0.0:1787826337.131865:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16938 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.131876:0:68279:0:(rw26.c:875:ll_write_end()) pos 69378048, len 4096, copied 4096 00000080:00200000:0.0:1787826337.131880:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000ee625d3e[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16938] 00000080:00200000:0.0:1787826337.131883:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000047d1d50f 17ffffc0001001 3:0 ffff907d91654cb0 16938 lru 00000080:00200000:0.0:1787826337.131888:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000005d30f478 16938: 1< 0 - - > 2< 69378048 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.131890:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000ee625d3e 00000080:00200000:0.0:1787826337.131891:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 43. 00000080:00200000:0.0:1787826337.131892:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16939 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.131904:0:68279:0:(rw26.c:875:ll_write_end()) pos 69382144, len 4096, copied 4096 00000080:00200000:0.0:1787826337.131908:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000023dd444c[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16939] 00000080:00200000:0.0:1787826337.131911:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000033c3489d 17ffffc0001001 3:0 ffff907d916549f8 16939 lru 00000080:00200000:0.0:1787826337.131916:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000035b339f9 16939: 1< 0 - - > 2< 69382144 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.131918:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000023dd444c 00000080:00200000:0.0:1787826337.131919:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 44. 00000080:00200000:0.0:1787826337.131921:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16940 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.131935:0:68279:0:(rw26.c:875:ll_write_end()) pos 69386240, len 4096, copied 4096 00000080:00200000:0.0:1787826337.131940:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000a5e2f717[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16940] 00000080:00200000:0.0:1787826337.131942:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000b112a8b0 17ffffc0001001 3:0 ffff907d91654570 16940 lru 00000080:00200000:0.0:1787826337.131947:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000002d6869bc 16940: 1< 0 - - > 2< 69386240 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.131949:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000a5e2f717 00000080:00200000:0.0:1787826337.131950:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 45. 00000080:00200000:0.0:1787826337.131952:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16941 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.131964:0:68279:0:(rw26.c:875:ll_write_end()) pos 69390336, len 4096, copied 4096 00000080:00200000:0.0:1787826337.131968:0:68279:0:(rw26.c:933:ll_write_end()) page@000000006dc2074c[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16941] 00000080:00200000:0.0:1787826337.131971:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000ed3fec1f 17ffffc0001001 3:0 ffff907d91654828 16941 lru 00000080:00200000:0.0:1787826337.131976:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000009431f7a2 16941: 1< 0 - - > 2< 69390336 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.131978:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000006dc2074c 00000080:00200000:0.0:1787826337.131979:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 46. 00000080:00200000:0.0:1787826337.131981:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16942 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.131994:0:68279:0:(rw26.c:875:ll_write_end()) pos 69394432, len 4096, copied 4096 00000080:00200000:0.0:1787826337.131998:0:68279:0:(rw26.c:933:ll_write_end()) page@000000003881744b[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16942] 00000080:00200000:0.0:1787826337.132001:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000085623566 17ffffc0001001 3:0 ffff907d91654d98 16942 lru 00000080:00200000:0.0:1787826337.132006:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000005ce4373e 16942: 1< 0 - - > 2< 69394432 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.132009:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000003881744b 00000080:00200000:0.0:1787826337.132010:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 47. 00000080:00200000:0.0:1787826337.132012:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16943 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.132036:0:68279:0:(rw26.c:875:ll_write_end()) pos 69398528, len 4096, copied 4096 00000080:00200000:0.0:1787826337.132040:0:68279:0:(rw26.c:933:ll_write_end()) page@000000006743c0d9[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16943] 00000080:00200000:0.0:1787826337.132043:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000b50bac23 17ffffc0001001 3:0 ffff907d91654658 16943 lru 00000080:00200000:0.0:1787826337.132047:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000025daa485 16943: 1< 0 - - > 2< 69398528 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.132049:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000006743c0d9 00000080:00200000:0.0:1787826337.132050:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 48. 00000080:00200000:0.0:1787826337.132052:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16944 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.132065:0:68279:0:(rw26.c:875:ll_write_end()) pos 69402624, len 4096, copied 4096 00000080:00200000:0.0:1787826337.132069:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000fc174a1d[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16944] 00000080:00200000:0.0:1787826337.132072:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000004b6a124d 17ffffc0001001 3:0 ffff907d916540e8 16944 lru 00000080:00200000:0.0:1787826337.132077:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000008d741a09 16944: 1< 0 - - > 2< 69402624 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.132079:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000fc174a1d 00000080:00200000:0.0:1787826337.132079:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 49. 00000080:00200000:0.0:1787826337.132081:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16945 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.132093:0:68279:0:(rw26.c:875:ll_write_end()) pos 69406720, len 4096, copied 4096 00000080:00200000:0.0:1787826337.132096:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000022b55b98[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16945] 00000080:00200000:0.0:1787826337.132099:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000007d855a24 17ffffc0001001 3:0 ffff907d91654910 16945 lru 00000080:00200000:0.0:1787826337.132104:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000065fd3887 16945: 1< 0 - - > 2< 69406720 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.132106:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000022b55b98 00000080:00200000:0.0:1787826337.132107:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 50. 00000080:00200000:0.0:1787826337.132109:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16946 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.132122:0:68279:0:(rw26.c:875:ll_write_end()) pos 69410816, len 4096, copied 4096 00000080:00200000:0.0:1787826337.132126:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000ff9ff840[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16946] 00000080:00200000:0.0:1787826337.132128:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000021bfcfa6 17ffffc0001001 3:0 ffff907d916541d0 16946 lru 00000080:00200000:0.0:1787826337.132134:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000000d2f09f1 16946: 1< 0 - - > 2< 69410816 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.132136:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000ff9ff840 00000080:00200000:0.0:1787826337.132137:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 51. 00000080:00200000:0.0:1787826337.132139:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16947 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.132155:0:68279:0:(rw26.c:875:ll_write_end()) pos 69414912, len 4096, copied 4096 00000080:00200000:0.0:1787826337.132159:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000212d66ca[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16947] 00000080:00200000:0.0:1787826337.132162:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000010b165a4 17ffffc0001001 3:0 ffff907d854ad1d0 16947 lru 00000080:00200000:0.0:1787826337.132168:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000003ec80fc4 16947: 1< 0 - - > 2< 69414912 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.132169:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000212d66ca 00000080:00200000:0.0:1787826337.132170:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 52. 00000080:00200000:0.0:1787826337.132172:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16948 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.132906:0:68279:0:(rw26.c:875:ll_write_end()) pos 69419008, len 4096, copied 4096 00000080:00200000:0.0:1787826337.132914:0:68279:0:(rw26.c:933:ll_write_end()) page@000000000ff319d0[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16948] 00000080:00200000:0.0:1787826337.132917:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000008888fc2a 17ffffc0001001 3:0 ffff907d854ad2b8 16948 lru 00000080:00200000:0.0:1787826337.132923:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000900b12e4 16948: 1< 0 - - > 2< 69419008 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.132926:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000000ff319d0 00000080:00200000:0.0:1787826337.132927:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 53. 00000080:00200000:0.0:1787826337.132929:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16949 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.132945:0:68279:0:(rw26.c:875:ll_write_end()) pos 69423104, len 4096, copied 4096 00000080:00200000:0.0:1787826337.132949:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000071e83f47[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16949] 00000080:00200000:0.0:1787826337.132952:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000050446156 17ffffc0001001 3:0 ffff907d854ad000 16949 lru 00000080:00200000:0.0:1787826337.132957:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000575ff086 16949: 1< 0 - - > 2< 69423104 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.132959:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000071e83f47 00000080:00200000:0.0:1787826337.132960:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 54. 00000080:00200000:0.0:1787826337.132962:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16950 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.132981:0:68279:0:(rw26.c:875:ll_write_end()) pos 69427200, len 4096, copied 4096 00000080:00200000:0.0:1787826337.132985:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000056aef2e9[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16950] 00000080:00200000:0.0:1787826337.132987:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000040ae60b 17ffffc0001021 2:0 ffff907d854adae0 16950 lru 00000080:00200000:0.0:1787826337.132992:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000000796dc98 16950: 1< 0 - - > 2< 69427200 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.132994:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000056aef2e9 00000080:00200000:0.0:1787826337.132995:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 55. 00000080:00200000:0.0:1787826337.132997:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16951 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.133011:0:68279:0:(rw26.c:875:ll_write_end()) pos 69431296, len 4096, copied 4096 00000080:00200000:0.0:1787826337.133015:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000817e685d[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16951] 00000080:00200000:0.0:1787826337.133018:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000040573b91 17ffffc0001001 3:0 ffff907d854ade80 16951 lru 00000080:00200000:0.0:1787826337.133024:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000e9ccacd0 16951: 1< 0 - - > 2< 69431296 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.133035:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000817e685d 00000080:00200000:0.0:1787826337.133037:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 56. 00000080:00200000:0.0:1787826337.133039:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16952 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.133054:0:68279:0:(rw26.c:875:ll_write_end()) pos 69435392, len 4096, copied 4096 00000080:00200000:0.0:1787826337.133058:0:68279:0:(rw26.c:933:ll_write_end()) page@000000007cb8656a[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16952] 00000080:00200000:0.0:1787826337.133061:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000375c548b 17ffffc0001001 3:0 ffff907d854ad3a0 16952 lru 00000080:00200000:0.0:1787826337.133065:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000001bc959e5 16952: 1< 0 - - > 2< 69435392 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.133067:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000007cb8656a 00000080:00200000:0.0:1787826337.133068:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 57. 00000080:00200000:0.0:1787826337.133069:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16953 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.133082:0:68279:0:(rw26.c:875:ll_write_end()) pos 69439488, len 4096, copied 4096 00000080:00200000:0.0:1787826337.133086:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000a002cf0d[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16953] 00000080:00200000:0.0:1787826337.133088:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000000cb112d 17ffffc0001001 3:0 ffff907d854ad488 16953 lru 00000080:00200000:0.0:1787826337.133094:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000f9497856 16953: 1< 0 - - > 2< 69439488 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.133096:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000a002cf0d 00000080:00200000:0.0:1787826337.133097:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 58. 00000080:00200000:0.0:1787826337.133098:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16954 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.133113:0:68279:0:(rw26.c:875:ll_write_end()) pos 69443584, len 4096, copied 4096 00000080:00200000:0.0:1787826337.133117:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000901d8816[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16954] 00000080:00200000:0.0:1787826337.133119:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000006e4d38f5 17ffffc0001001 3:0 ffff907d854ad740 16954 lru 00000080:00200000:0.0:1787826337.133124:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000003c6642a3 16954: 1< 0 - - > 2< 69443584 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.133126:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000901d8816 00000080:00200000:0.0:1787826337.133127:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 59. 00000080:00200000:0.0:1787826337.133129:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16955 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.133142:0:68279:0:(rw26.c:875:ll_write_end()) pos 69447680, len 4096, copied 4096 00000080:00200000:0.0:1787826337.133146:0:68279:0:(rw26.c:933:ll_write_end()) page@000000006f836002[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16955] 00000080:00200000:0.0:1787826337.133149:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000d8259fe7 17ffffc0001001 3:0 ffff907d854adbc8 16955 lru 00000080:00200000:0.0:1787826337.133154:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000019ad088f 16955: 1< 0 - - > 2< 69447680 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.133156:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000006f836002 00000080:00200000:0.0:1787826337.133157:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 60. 00000080:00200000:0.0:1787826337.133158:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16956 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.133171:0:68279:0:(rw26.c:875:ll_write_end()) pos 69451776, len 4096, copied 4096 00000080:00200000:0.0:1787826337.134215:0:68279:0:(rw26.c:933:ll_write_end()) page@000000004bf0ace5[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16956] 00000080:00200000:0.0:1787826337.134222:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000ed34932d 17ffffc0001001 3:0 ffff907d854adcb0 16956 lru 00000080:00200000:0.0:1787826337.134228:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000cebec681 16956: 1< 0 - - > 2< 69451776 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.134231:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000004bf0ace5 00000080:00200000:0.0:1787826337.134232:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 61. 00000080:00200000:0.0:1787826337.134237:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16957 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.134265:0:68279:0:(rw26.c:875:ll_write_end()) pos 69455872, len 4096, copied 4096 00000080:00200000:0.0:1787826337.134270:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000561cc873[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16957] 00000080:00200000:0.0:1787826337.134273:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000098d564df 17ffffc0001001 3:0 ffff907d854ad9f8 16957 lru 00000080:00200000:0.0:1787826337.134278:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000070f0e668 16957: 1< 0 - - > 2< 69455872 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.134280:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000561cc873 00000080:00200000:0.0:1787826337.134281:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 62. 00000080:00200000:0.0:1787826337.134283:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16958 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.134427:0:68279:0:(rw26.c:875:ll_write_end()) pos 69459968, len 4096, copied 4096 00000080:00200000:0.0:1787826337.134432:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000b4cb6126[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16958] 00000080:00200000:0.0:1787826337.134435:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000cae5acb9 17ffffc0001001 3:0 ffff907d854ad570 16958 lru 00000080:00200000:0.0:1787826337.134440:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000001af763c9 16958: 1< 0 - - > 2< 69459968 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.134442:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000b4cb6126 00000080:00200000:0.0:1787826337.134443:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 63. 00000080:00200000:0.0:1787826337.134445:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16959 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.134454:0:68279:0:(rw26.c:875:ll_write_end()) pos 69464064, len 4096, copied 4096 00000080:00200000:0.0:1787826337.134457:0:68279:0:(rw26.c:933:ll_write_end()) page@000000009e124228[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16959] 00000080:00200000:0.0:1787826337.134460:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000014bd1fea 17ffffc0001001 3:0 ffff907d854ad828 16959 lru 00000080:00200000:0.0:1787826337.134465:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000ee0c369e 16959: 1< 0 - - > 2< 69464064 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.134467:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000009e124228 00000080:00200000:0.0:1787826337.134468:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 64. 00000080:00200000:0.0:1787826337.134470:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16960 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.134488:0:68279:0:(rw26.c:875:ll_write_end()) pos 69468160, len 4096, copied 4096 00000080:00200000:0.0:1787826337.134492:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000ed5a1c56[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16960] 00000080:00200000:0.0:1787826337.134494:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000207b47c1 17ffffc0001001 3:0 ffff907d854add98 16960 lru 00000080:00200000:0.0:1787826337.134500:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000000bf6c345 16960: 1< 0 - - > 2< 69468160 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.134501:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000ed5a1c56 00000080:00200000:0.0:1787826337.134502:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 65. 00000080:00200000:0.0:1787826337.134504:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16961 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.134514:0:68279:0:(rw26.c:875:ll_write_end()) pos 69472256, len 4096, copied 4096 00000080:00200000:0.0:1787826337.134518:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000be788383[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16961] 00000080:00200000:0.0:1787826337.134521:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000929ddf12 17ffffc0001001 3:0 ffff907d854ad658 16961 lru 00000080:00200000:0.0:1787826337.134526:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000181bb651 16961: 1< 0 - - > 2< 69472256 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.134528:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000be788383 00000080:00200000:0.0:1787826337.134529:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 66. 00000080:00200000:0.0:1787826337.134530:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16962 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.134539:0:68279:0:(rw26.c:875:ll_write_end()) pos 69476352, len 4096, copied 4096 00000080:00200000:0.0:1787826337.134543:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000043a6a196[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16962] 00000080:00200000:0.0:1787826337.134546:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000000186d854 17ffffc0001001 3:0 ffff907d854ad0e8 16962 lru 00000080:00200000:0.0:1787826337.134551:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000030585f7f 16962: 1< 0 - - > 2< 69476352 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.134552:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000043a6a196 00000080:00200000:0.0:1787826337.134553:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 67. 00000080:00200000:0.0:1787826337.134554:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16963 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.134563:0:68279:0:(rw26.c:875:ll_write_end()) pos 69480448, len 4096, copied 4096 00000080:00200000:0.0:1787826337.134567:0:68279:0:(rw26.c:933:ll_write_end()) page@000000000d8ea3c8[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16963] 00000080:00200000:0.0:1787826337.134569:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000492cb5b7 17ffffc0001001 3:0 ffff907d854ad910 16963 lru 00000080:00200000:0.0:1787826337.134574:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000009d7d8773 16963: 1< 0 - - > 2< 69480448 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.134576:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000000d8ea3c8 00000080:00200000:0.0:1787826337.134577:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 68. 00000080:00200000:0.0:1787826337.134578:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16964 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.134588:0:68279:0:(rw26.c:875:ll_write_end()) pos 69484544, len 4096, copied 4096 00000080:00200000:0.0:1787826337.134592:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000cdfc0fca[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16964] 00000080:00200000:0.0:1787826337.134595:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000022ee7a69 17ffffc0001001 3:0 ffff907d96009910 16964 lru 00000080:00200000:0.0:1787826337.134600:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000ac02645c 16964: 1< 0 - - > 2< 69484544 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.134602:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000cdfc0fca 00000080:00200000:0.0:1787826337.134603:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 69. 00000080:00200000:0.0:1787826337.134735:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16965 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.134759:0:68279:0:(rw26.c:875:ll_write_end()) pos 69488640, len 4096, copied 4096 00000080:00200000:0.0:1787826337.134763:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000011d16d2a[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16965] 00000080:00200000:0.0:1787826337.134766:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000001383607c 17ffffc0001021 2:0 ffff907d960091d0 16965 lru 00000080:00200000:0.0:1787826337.134770:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000007279f5f0 16965: 1< 0 - - > 2< 69488640 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.134773:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000011d16d2a 00000080:00200000:0.0:1787826337.134774:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 70. 00000080:00200000:0.0:1787826337.134776:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16966 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.134789:0:68279:0:(rw26.c:875:ll_write_end()) pos 69492736, len 4096, copied 4096 00000080:00200000:0.0:1787826337.134794:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000cb9f2b39[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16966] 00000080:00200000:0.0:1787826337.134796:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000007acc1e74 17ffffc0001001 3:0 ffff907d960092b8 16966 lru 00000080:00200000:0.0:1787826337.134801:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000bcf9c10c 16966: 1< 0 - - > 2< 69492736 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.134804:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000cb9f2b39 00000080:00200000:0.0:1787826337.134805:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 71. 00000080:00200000:0.0:1787826337.134806:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16967 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.134819:0:68279:0:(rw26.c:875:ll_write_end()) pos 69496832, len 4096, copied 4096 00000080:00200000:0.0:1787826337.134822:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000e472a008[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16967] 00000080:00200000:0.0:1787826337.134825:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000b11052a3 17ffffc0001001 3:0 ffff907d96009000 16967 lru 00000080:00200000:0.0:1787826337.134830:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000f91fbe7a 16967: 1< 0 - - > 2< 69496832 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.134832:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000e472a008 00000080:00200000:0.0:1787826337.134833:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 72. 00000080:00200000:0.0:1787826337.134835:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16968 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.134848:0:68279:0:(rw26.c:875:ll_write_end()) pos 69500928, len 4096, copied 4096 00000080:00200000:0.0:1787826337.134852:0:68279:0:(rw26.c:933:ll_write_end()) page@000000004ea6ae62[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16968] 00000080:00200000:0.0:1787826337.134855:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000007a7cdf47 17ffffc0001001 3:0 ffff907d96009ae0 16968 lru 00000080:00200000:0.0:1787826337.134860:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000707d951f 16968: 1< 0 - - > 2< 69500928 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.134863:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000004ea6ae62 00000080:00200000:0.0:1787826337.134864:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 73. 00000080:00200000:0.0:1787826337.134866:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16969 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.134883:0:68279:0:(rw26.c:875:ll_write_end()) pos 69505024, len 4096, copied 4096 00000080:00200000:0.0:1787826337.134887:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000014ff3383[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16969] 00000080:00200000:0.0:1787826337.134890:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000050aa9ce7 17ffffc0001001 3:0 ffff907d96009e80 16969 lru 00000080:00200000:0.0:1787826337.134894:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000e30e8885 16969: 1< 0 - - > 2< 69505024 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.134896:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000014ff3383 00000080:00200000:0.0:1787826337.134897:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 74. 00000080:00200000:0.0:1787826337.134899:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16970 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.134911:0:68279:0:(rw26.c:875:ll_write_end()) pos 69509120, len 4096, copied 4096 00000080:00200000:0.0:1787826337.134915:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000ccb29b96[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16970] 00000080:00200000:0.0:1787826337.134917:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000f4b04c6d 17ffffc0001001 3:0 ffff907d960093a0 16970 lru 00000080:00200000:0.0:1787826337.134923:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000703f51eb 16970: 1< 0 - - > 2< 69509120 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.134925:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000ccb29b96 00000080:00200000:0.0:1787826337.134926:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 75. 00000080:00200000:0.0:1787826337.134928:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16971 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.134941:0:68279:0:(rw26.c:875:ll_write_end()) pos 69513216, len 4096, copied 4096 00000080:00200000:0.0:1787826337.134945:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000d4829417[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16971] 00000080:00200000:0.0:1787826337.134948:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000239ffbdd 17ffffc0001001 3:0 ffff907d96009488 16971 lru 00000080:00200000:0.0:1787826337.134953:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000581a2355 16971: 1< 0 - - > 2< 69513216 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.134955:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000d4829417 00000080:00200000:0.0:1787826337.134956:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 76. 00000080:00200000:0.0:1787826337.134958:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16972 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.134971:0:68279:0:(rw26.c:875:ll_write_end()) pos 69517312, len 4096, copied 4096 00000080:00200000:0.0:1787826337.134975:0:68279:0:(rw26.c:933:ll_write_end()) page@000000004ac398ef[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16972] 00000080:00200000:0.0:1787826337.134978:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000e15cb0bc 17ffffc0001001 3:0 ffff907d96009740 16972 lru 00000080:00200000:0.0:1787826337.134983:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000f5cdda6d 16972: 1< 0 - - > 2< 69517312 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.134985:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000004ac398ef 00000080:00200000:0.0:1787826337.134986:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 77. 00000080:00200000:0.0:1787826337.134988:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16973 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.135000:0:68279:0:(rw26.c:875:ll_write_end()) pos 69521408, len 4096, copied 4096 00000080:00200000:0.0:1787826337.135004:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000014fab045[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16973] 00000080:00200000:0.0:1787826337.135007:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000009186b553 17ffffc0001001 3:0 ffff907d96009bc8 16973 lru 00000080:00200000:0.0:1787826337.135011:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000e227276a 16973: 1< 0 - - > 2< 69521408 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.135013:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000014fab045 00000080:00200000:0.0:1787826337.135014:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 78. 00000080:00200000:0.0:1787826337.135016:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16974 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.135043:0:68279:0:(rw26.c:875:ll_write_end()) pos 69525504, len 4096, copied 4096 00000080:00200000:0.0:1787826337.135048:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000e85374fb[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16974] 00000080:00200000:0.0:1787826337.135050:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000cfb50171 17ffffc0001001 3:0 ffff907d96009cb0 16974 lru 00000080:00200000:0.0:1787826337.135055:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000f34b9acd 16974: 1< 0 - - > 2< 69525504 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.135057:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000e85374fb 00000080:00200000:0.0:1787826337.135058:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 79. 00000080:00200000:0.0:1787826337.135059:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16975 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.135073:0:68279:0:(rw26.c:875:ll_write_end()) pos 69529600, len 4096, copied 4096 00000080:00200000:0.0:1787826337.135077:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000065c442d4[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16975] 00000080:00200000:0.0:1787826337.135080:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000003f7dc2d3 17ffffc0001001 3:0 ffff907d960099f8 16975 lru 00000080:00200000:0.0:1787826337.135085:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000044eee189 16975: 1< 0 - - > 2< 69529600 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.135087:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000065c442d4 00000080:00200000:0.0:1787826337.135088:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 80. 00000080:00200000:0.0:1787826337.135090:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16976 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.135104:0:68279:0:(rw26.c:875:ll_write_end()) pos 69533696, len 4096, copied 4096 00000080:00200000:0.0:1787826337.135108:0:68279:0:(rw26.c:933:ll_write_end()) page@000000000806ac56[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16976] 00000080:00200000:0.0:1787826337.135111:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000d3a82545 17ffffc0001001 3:0 ffff907d96009570 16976 lru 00000080:00200000:0.0:1787826337.135116:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000009492f68b 16976: 1< 0 - - > 2< 69533696 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.135118:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000000806ac56 00000080:00200000:0.0:1787826337.135119:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 81. 00000080:00200000:0.0:1787826337.135121:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16977 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.135134:0:68279:0:(rw26.c:875:ll_write_end()) pos 69537792, len 4096, copied 4096 00000080:00200000:0.0:1787826337.135138:0:68279:0:(rw26.c:933:ll_write_end()) page@000000001a78b676[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16977] 00000080:00200000:0.0:1787826337.135141:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000ffffd91a 17ffffc0001001 3:0 ffff907d96009828 16977 lru 00000080:00200000:0.0:1787826337.135147:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000439ea79f 16977: 1< 0 - - > 2< 69537792 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.135149:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000001a78b676 00000080:00200000:0.0:1787826337.135150:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 82. 00000080:00200000:0.0:1787826337.135152:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16978 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.135164:0:68279:0:(rw26.c:875:ll_write_end()) pos 69541888, len 4096, copied 4096 00000080:00200000:0.0:1787826337.135169:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000f3ca924d[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16978] 00000080:00200000:0.0:1787826337.135172:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000004a9ad1eb 17ffffc0001001 3:0 ffff907d96009d98 16978 lru 00000080:00200000:0.0:1787826337.135253:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000007c27c081 16978: 1< 0 - - > 2< 69541888 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.135256:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000f3ca924d 00000080:00200000:0.0:1787826337.135257:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 83. 00000080:00200000:0.0:1787826337.135259:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16979 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.135286:0:68279:0:(rw26.c:875:ll_write_end()) pos 69545984, len 4096, copied 4096 00000080:00200000:0.0:1787826337.135780:0:68279:0:(rw26.c:933:ll_write_end()) page@000000003b9f0819[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16979] 00000080:00200000:0.0:1787826337.135783:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000ffa81c4f 17ffffc0001001 3:0 ffff907d96009658 16979 lru 00000080:00200000:0.0:1787826337.135788:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000dd561af6 16979: 1< 0 - - > 2< 69545984 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.135790:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000003b9f0819 00000080:00200000:0.0:1787826337.135792:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 84. 00000080:00200000:0.0:1787826337.135794:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16980 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.135818:0:68279:0:(rw26.c:875:ll_write_end()) pos 69550080, len 4096, copied 4096 00000080:00200000:0.0:1787826337.135822:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000354a30ab[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16980] 00000080:00200000:0.0:1787826337.135825:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000af6910f6 17ffffc0001021 2:0 ffff907d960090e8 16980 lru 00000080:00200000:0.0:1787826337.135830:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000001779e88b 16980: 1< 0 - - > 2< 69550080 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.135832:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000354a30ab 00000080:00200000:0.0:1787826337.135832:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 85. 00000080:00200000:0.0:1787826337.135834:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16981 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.135849:0:68279:0:(rw26.c:875:ll_write_end()) pos 69554176, len 4096, copied 4096 00000080:00200000:0.0:1787826337.135853:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000c3b2d81f[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16981] 00000080:00200000:0.0:1787826337.135855:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000036432292 17ffffc0001001 3:0 ffff907d88f40488 16981 lru 00000080:00200000:0.0:1787826337.135860:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000b615955d 16981: 1< 0 - - > 2< 69554176 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.135862:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000c3b2d81f 00000080:00200000:0.0:1787826337.135863:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 86. 00000080:00200000:0.0:1787826337.135865:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16982 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.135879:0:68279:0:(rw26.c:875:ll_write_end()) pos 69558272, len 4096, copied 4096 00000080:00200000:0.0:1787826337.135883:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000053e00838[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16982] 00000080:00200000:0.0:1787826337.135886:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000a108ff9d 17ffffc0001001 3:0 ffff907d88f40740 16982 lru 00000080:00200000:0.0:1787826337.135890:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000064bf62a2 16982: 1< 0 - - > 2< 69558272 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.135892:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000053e00838 00000080:00200000:0.0:1787826337.135893:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 87. 00000080:00200000:0.0:1787826337.135896:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16983 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.135909:0:68279:0:(rw26.c:875:ll_write_end()) pos 69562368, len 4096, copied 4096 00000080:00200000:0.0:1787826337.135914:0:68279:0:(rw26.c:933:ll_write_end()) page@000000005cc32be8[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16983] 00000080:00200000:0.0:1787826337.135916:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000047639ff8 17ffffc0001001 3:0 ffff907d88f40bc8 16983 lru 00000080:00200000:0.0:1787826337.135921:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000082aa12d3 16983: 1< 0 - - > 2< 69562368 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.135923:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000005cc32be8 00000080:00200000:0.0:1787826337.135924:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 88. 00000080:00200000:0.0:1787826337.135926:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16984 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.135939:0:68279:0:(rw26.c:875:ll_write_end()) pos 69566464, len 4096, copied 4096 00000080:00200000:0.0:1787826337.135943:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000d46f5d35[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16984] 00000080:00200000:0.0:1787826337.135946:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000d4a77910 17ffffc0001001 3:0 ffff907d88f40cb0 16984 lru 00000080:00200000:0.0:1787826337.135951:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000004a9be5f2 16984: 1< 0 - - > 2< 69566464 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.135954:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000d46f5d35 00000080:00200000:0.0:1787826337.135955:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 89. 00000080:00200000:0.0:1787826337.135957:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16985 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.135972:0:68279:0:(rw26.c:875:ll_write_end()) pos 69570560, len 4096, copied 4096 00000080:00200000:0.0:1787826337.135976:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000099e16f37[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16985] 00000080:00200000:0.0:1787826337.135979:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000001d35012 17ffffc0001001 3:0 ffff907d88f409f8 16985 lru 00000080:00200000:0.0:1787826337.135984:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000207c12fa 16985: 1< 0 - - > 2< 69570560 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.135986:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000099e16f37 00000080:00200000:0.0:1787826337.135987:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 90. 00000080:00200000:0.0:1787826337.135989:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16986 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.136004:0:68279:0:(rw26.c:875:ll_write_end()) pos 69574656, len 4096, copied 4096 00000080:00200000:0.0:1787826337.136007:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000074940f2e[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16986] 00000080:00200000:0.0:1787826337.136010:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000009ceb855c 17ffffc0001001 3:0 ffff907d88f40570 16986 lru 00000080:00200000:0.0:1787826337.136015:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000048d9e7d4 16986: 1< 0 - - > 2< 69574656 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.136017:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000074940f2e 00000080:00200000:0.0:1787826337.136018:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 91. 00000080:00200000:0.0:1787826337.136020:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16987 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.136042:0:68279:0:(rw26.c:875:ll_write_end()) pos 69578752, len 4096, copied 4096 00000080:00200000:0.0:1787826337.136047:0:68279:0:(rw26.c:933:ll_write_end()) page@000000006d388b90[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16987] 00000080:00200000:0.0:1787826337.136050:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000a953d283 17ffffc0001001 3:0 ffff907d88f40828 16987 lru 00000080:00200000:0.0:1787826337.136055:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000040c696f0 16987: 1< 0 - - > 2< 69578752 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.136056:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000006d388b90 00000080:00200000:0.0:1787826337.136057:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 92. 00000080:00200000:0.0:1787826337.136059:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16988 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.136073:0:68279:0:(rw26.c:875:ll_write_end()) pos 69582848, len 4096, copied 4096 00000080:00200000:0.0:1787826337.136077:0:68279:0:(rw26.c:933:ll_write_end()) page@000000008cf3a78e[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16988] 00000080:00200000:0.0:1787826337.136079:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000868d6bf3 17ffffc0001001 3:0 ffff907d88f40d98 16988 lru 00000080:00200000:0.0:1787826337.136084:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000015d1a4ac 16988: 1< 0 - - > 2< 69582848 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.136087:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000008cf3a78e 00000080:00200000:0.0:1787826337.136087:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 93. 00000080:00200000:0.0:1787826337.136089:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16989 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.136104:0:68279:0:(rw26.c:875:ll_write_end()) pos 69586944, len 4096, copied 4096 00000080:00200000:0.0:1787826337.136108:0:68279:0:(rw26.c:933:ll_write_end()) page@000000008d266272[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16989] 00000080:00200000:0.0:1787826337.136111:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000005486c872 17ffffc0001001 3:0 ffff907d88f40658 16989 lru 00000080:00200000:0.0:1787826337.136115:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000007e693567 16989: 1< 0 - - > 2< 69586944 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.136118:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000008d266272 00000080:00200000:0.0:1787826337.136119:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 94. 00000080:00200000:0.0:1787826337.136120:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16990 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.136133:0:68279:0:(rw26.c:875:ll_write_end()) pos 69591040, len 4096, copied 4096 00000080:00200000:0.0:1787826337.136137:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000d0e8a9cb[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16990] 00000080:00200000:0.0:1787826337.136140:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000197e502d 17ffffc0001001 3:0 ffff907d88f400e8 16990 lru 00000080:00200000:0.0:1787826337.136146:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000007c441517 16990: 1< 0 - - > 2< 69591040 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.136148:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000d0e8a9cb 00000080:00200000:0.0:1787826337.136148:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 95. 00000080:00200000:0.0:1787826337.136150:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16991 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.136164:0:68279:0:(rw26.c:875:ll_write_end()) pos 69595136, len 4096, copied 4096 00000080:00200000:0.0:1787826337.136168:0:68279:0:(rw26.c:933:ll_write_end()) page@000000003b76becb[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16991] 00000080:00200000:0.0:1787826337.136170:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000075cebc7d 17ffffc0001001 3:0 ffff907d88f40910 16991 lru 00000080:00200000:0.0:1787826337.136182:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000e080ed58 16991: 1< 0 - - > 2< 69595136 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.136185:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000003b76becb 00000080:00200000:0.0:1787826337.136186:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 96. 00000080:00200000:0.0:1787826337.136188:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16992 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.136203:0:68279:0:(rw26.c:875:ll_write_end()) pos 69599232, len 4096, copied 4096 00000080:00200000:0.0:1787826337.136207:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000061b9e798[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16992] 00000080:00200000:0.0:1787826337.136210:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000fdedd7bc 17ffffc0001001 3:0 ffff907d88f401d0 16992 lru 00000080:00200000:0.0:1787826337.136216:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000f1a695e3 16992: 1< 0 - - > 2< 69599232 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.136218:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000061b9e798 00000080:00200000:0.0:1787826337.136219:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 97. 00000080:00200000:0.0:1787826337.136220:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16993 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.136229:0:68279:0:(rw26.c:875:ll_write_end()) pos 69603328, len 4096, copied 4096 00000080:00200000:0.0:1787826337.136233:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000c6726e5d[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16993] 00000080:00200000:0.0:1787826337.136235:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000ac3c67cd 17ffffc0001001 3:0 ffff907d88f402b8 16993 lru 00000080:00200000:0.0:1787826337.136240:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000630ef77a 16993: 1< 0 - - > 2< 69603328 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.136242:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000c6726e5d 00000080:00200000:0.0:1787826337.136243:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 98. 00000080:00200000:0.0:1787826337.136245:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16994 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.136253:0:68279:0:(rw26.c:875:ll_write_end()) pos 69607424, len 4096, copied 4096 00000080:00200000:0.0:1787826337.136257:0:68279:0:(rw26.c:933:ll_write_end()) page@000000004709f37d[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16994] 00000080:00200000:0.0:1787826337.136259:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000002a09f2f5 17ffffc0001001 3:0 ffff907d88f40000 16994 lru 00000080:00200000:0.0:1787826337.136265:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000005952719c 16994: 1< 0 - - > 2< 69607424 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.136267:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000004709f37d 00000080:00200000:0.0:1787826337.136268:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 99. 00000080:00200000:0.0:1787826337.136269:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16995 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.136282:0:68279:0:(rw26.c:875:ll_write_end()) pos 69611520, len 4096, copied 4096 00000080:00200000:0.0:1787826337.136285:0:68279:0:(rw26.c:933:ll_write_end()) page@000000009cc1e334[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16995] 00000080:00200000:0.0:1787826337.137900:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000009a6fffbd 17ffffc0001021 2:0 ffff907d88f40ae0 16995 lru 00000080:00200000:0.0:1787826337.137910:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000096515fd5 16995: 1< 0 - - > 2< 69611520 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.137913:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000009cc1e334 00000080:00200000:0.0:1787826337.137914:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 100. 00000080:00200000:0.0:1787826337.137921:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16996 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.137966:0:68279:0:(rw26.c:875:ll_write_end()) pos 69615616, len 4096, copied 4096 00000080:00200000:0.0:1787826337.137973:0:68279:0:(rw26.c:933:ll_write_end()) page@000000001a1e0688[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16996] 00000080:00200000:0.0:1787826337.137976:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000b1764662 17ffffc0001001 3:0 ffff907d88f40e80 16996 lru 00000080:00200000:0.0:1787826337.137981:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000fb66c08e 16996: 1< 0 - - > 2< 69615616 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.137983:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000001a1e0688 00000080:00200000:0.0:1787826337.137984:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 101. 00000080:00200000:0.0:1787826337.137986:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16997 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138001:0:68279:0:(rw26.c:875:ll_write_end()) pos 69619712, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138005:0:68279:0:(rw26.c:933:ll_write_end()) page@000000005cf397e9[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16997] 00000080:00200000:0.0:1787826337.138008:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000d509f303 17ffffc0001001 3:0 ffff907d88f403a0 16997 lru 00000080:00200000:0.0:1787826337.138013:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000034bd779a 16997: 1< 0 - - > 2< 69619712 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138015:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000005cf397e9 00000080:00200000:0.0:1787826337.138017:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 102. 00000080:00200000:0.0:1787826337.138018:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16998 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138043:0:68279:0:(rw26.c:875:ll_write_end()) pos 69623808, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138048:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000002a449e5[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16998] 00000080:00200000:0.0:1787826337.138050:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000085531397 17ffffc0001001 3:0 ffff907d83b0ad98 16998 lru 00000080:00200000:0.0:1787826337.138055:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000000a082379 16998: 1< 0 - - > 2< 69623808 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138057:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000002a449e5 00000080:00200000:0.0:1787826337.138058:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 103. 00000080:00200000:0.0:1787826337.138061:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 16999 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138075:0:68279:0:(rw26.c:875:ll_write_end()) pos 69627904, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138078:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000a5e08a03[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 16999] 00000080:00200000:0.0:1787826337.138081:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000d2de856d 17ffffc0001001 3:0 ffff907d83b0a658 16999 lru 00000080:00200000:0.0:1787826337.138085:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000607b3e5b 16999: 1< 0 - - > 2< 69627904 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138086:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000a5e08a03 00000080:00200000:0.0:1787826337.138087:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 104. 00000080:00200000:0.0:1787826337.138089:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17000 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138099:0:68279:0:(rw26.c:875:ll_write_end()) pos 69632000, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138102:0:68279:0:(rw26.c:933:ll_write_end()) page@000000005e4a0b36[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17000] 00000080:00200000:0.0:1787826337.138105:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000000101760c 17ffffc0001001 3:0 ffff907d83b0a0e8 17000 lru 00000080:00200000:0.0:1787826337.138108:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000ca67e447 17000: 1< 0 - - > 2< 69632000 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138110:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000005e4a0b36 00000080:00200000:0.0:1787826337.138111:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 105. 00000080:00200000:0.0:1787826337.138112:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17001 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138123:0:68279:0:(rw26.c:875:ll_write_end()) pos 69636096, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138127:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000c40011fd[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17001] 00000080:00200000:0.0:1787826337.138130:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000001407735a 17ffffc0001001 3:0 ffff907d83b0a910 17001 lru 00000080:00200000:0.0:1787826337.138135:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000052afc9d8 17001: 1< 0 - - > 2< 69636096 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138138:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000c40011fd 00000080:00200000:0.0:1787826337.138139:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 106. 00000080:00200000:0.0:1787826337.138141:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17002 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138153:0:68279:0:(rw26.c:875:ll_write_end()) pos 69640192, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138157:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000ad8a5208[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17002] 00000080:00200000:0.0:1787826337.138160:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000627af706 17ffffc0001001 3:0 ffff907d83b0a1d0 17002 lru 00000080:00200000:0.0:1787826337.138164:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000003d508904 17002: 1< 0 - - > 2< 69640192 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138166:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000ad8a5208 00000080:00200000:0.0:1787826337.138168:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 107. 00000080:00200000:0.0:1787826337.138169:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17003 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138183:0:68279:0:(rw26.c:875:ll_write_end()) pos 69644288, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138192:0:68279:0:(rw26.c:933:ll_write_end()) page@000000008f7f1ebb[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17003] 00000080:00200000:0.0:1787826337.138196:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000f3b9ce60 17ffffc0001001 3:0 ffff907d83b0a2b8 17003 lru 00000080:00200000:0.0:1787826337.138201:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000e7d20628 17003: 1< 0 - - > 2< 69644288 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138203:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000008f7f1ebb 00000080:00200000:0.0:1787826337.138204:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 108. 00000080:00200000:0.0:1787826337.138206:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17004 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138220:0:68279:0:(rw26.c:875:ll_write_end()) pos 69648384, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138224:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000074ca3719[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17004] 00000080:00200000:0.0:1787826337.138227:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000096945b42 17ffffc0001001 3:0 ffff907d83b0a000 17004 lru 00000080:00200000:0.0:1787826337.138232:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000442fa0e8 17004: 1< 0 - - > 2< 69648384 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138234:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000074ca3719 00000080:00200000:0.0:1787826337.138235:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 109. 00000080:00200000:0.0:1787826337.138237:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17005 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138251:0:68279:0:(rw26.c:875:ll_write_end()) pos 69652480, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138255:0:68279:0:(rw26.c:933:ll_write_end()) page@000000005457ff70[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17005] 00000080:00200000:0.0:1787826337.138258:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000007192e4ec 17ffffc0001001 3:0 ffff907d83b0aae0 17005 lru 00000080:00200000:0.0:1787826337.138263:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000048f0792e 17005: 1< 0 - - > 2< 69652480 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138265:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000005457ff70 00000080:00200000:0.0:1787826337.138266:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 110. 00000080:00200000:0.0:1787826337.138268:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17006 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138281:0:68279:0:(rw26.c:875:ll_write_end()) pos 69656576, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138285:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000034f4fee3[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17006] 00000080:00200000:0.0:1787826337.138288:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000000a24028c 17ffffc0001001 3:0 ffff907d83b0ae80 17006 lru 00000080:00200000:0.0:1787826337.138305:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000002a871fd4 17006: 1< 0 - - > 2< 69656576 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138307:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000034f4fee3 00000080:00200000:0.0:1787826337.138308:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 111. 00000080:00200000:0.0:1787826337.138310:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17007 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138325:0:68279:0:(rw26.c:875:ll_write_end()) pos 69660672, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138329:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000713c3aa3[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17007] 00000080:00200000:0.0:1787826337.138332:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000dcb363de 17ffffc0001001 3:0 ffff907d83b0a3a0 17007 lru 00000080:00200000:0.0:1787826337.138336:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000381ade6f 17007: 1< 0 - - > 2< 69660672 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138339:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000713c3aa3 00000080:00200000:0.0:1787826337.138340:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 112. 00000080:00200000:0.0:1787826337.138341:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17008 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138355:0:68279:0:(rw26.c:875:ll_write_end()) pos 69664768, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138359:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000b14a81a7[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17008] 00000080:00200000:0.0:1787826337.138362:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000004d5cde71 17ffffc0001001 3:0 ffff907d83b0a488 17008 lru 00000080:00200000:0.0:1787826337.138367:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000bc5f2475 17008: 1< 0 - - > 2< 69664768 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138369:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000b14a81a7 00000080:00200000:0.0:1787826337.138370:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 113. 00000080:00200000:0.0:1787826337.138372:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17009 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138386:0:68279:0:(rw26.c:875:ll_write_end()) pos 69668864, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138391:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000c4de335b[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17009] 00000080:00200000:0.0:1787826337.138393:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000098742bb8 17ffffc0001001 3:0 ffff907d83b0a740 17009 lru 00000080:00200000:0.0:1787826337.138398:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000371b6a63 17009: 1< 0 - - > 2< 69668864 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138400:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000c4de335b 00000080:00200000:0.0:1787826337.138401:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 114. 00000080:00200000:0.0:1787826337.138403:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17010 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138425:0:68279:0:(rw26.c:875:ll_write_end()) pos 69672960, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138429:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000081871dfa[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17010] 00000080:00200000:0.0:1787826337.138432:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000002683eb29 17ffffc0001021 2:0 ffff907d83b0abc8 17010 lru 00000080:00200000:0.0:1787826337.138437:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000002d4cad46 17010: 1< 0 - - > 2< 69672960 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138440:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000081871dfa 00000080:00200000:0.0:1787826337.138441:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 115. 00000080:00200000:0.0:1787826337.138443:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17011 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138457:0:68279:0:(rw26.c:875:ll_write_end()) pos 69677056, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138461:0:68279:0:(rw26.c:933:ll_write_end()) page@000000007aa257b2[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17011] 00000080:00200000:0.0:1787826337.138463:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000c8b5639f 17ffffc0001001 3:0 ffff907d83b0acb0 17011 lru 00000080:00200000:0.0:1787826337.138468:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000a14c90e3 17011: 1< 0 - - > 2< 69677056 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138470:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000007aa257b2 00000080:00200000:0.0:1787826337.138471:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 116. 00000080:00200000:0.0:1787826337.138473:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17012 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138485:0:68279:0:(rw26.c:875:ll_write_end()) pos 69681152, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138489:0:68279:0:(rw26.c:933:ll_write_end()) page@000000007ff8bf70[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17012] 00000080:00200000:0.0:1787826337.138491:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000bbcaf50d 17ffffc0001001 3:0 ffff907d83b0a9f8 17012 lru 00000080:00200000:0.0:1787826337.138496:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000008ac265cf 17012: 1< 0 - - > 2< 69681152 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138498:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000007ff8bf70 00000080:00200000:0.0:1787826337.138499:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 117. 00000080:00200000:0.0:1787826337.138501:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17013 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138512:0:68279:0:(rw26.c:875:ll_write_end()) pos 69685248, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138516:0:68279:0:(rw26.c:933:ll_write_end()) page@000000002bd1a6e3[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17013] 00000080:00200000:0.0:1787826337.138519:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000008c3ccddb 17ffffc0001001 3:0 ffff907d83b0a570 17013 lru 00000080:00200000:0.0:1787826337.138524:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000ffba46c1 17013: 1< 0 - - > 2< 69685248 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138526:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000002bd1a6e3 00000080:00200000:0.0:1787826337.138527:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 118. 00000080:00200000:0.0:1787826337.138529:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17014 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138541:0:68279:0:(rw26.c:875:ll_write_end()) pos 69689344, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138546:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000052b7773e[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17014] 00000080:00200000:0.0:1787826337.138548:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000f08ca5d3 17ffffc0001001 3:0 ffff907d83b0a828 17014 lru 00000080:00200000:0.0:1787826337.138552:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000041e9c3d9 17014: 1< 0 - - > 2< 69689344 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138554:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000052b7773e 00000080:00200000:0.0:1787826337.138555:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 119. 00000080:00200000:0.0:1787826337.138557:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17015 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138570:0:68279:0:(rw26.c:875:ll_write_end()) pos 69693440, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138574:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000053668cc1[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17015] 00000080:00200000:0.0:1787826337.138576:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000913d66dc 17ffffc0001001 3:0 ffff907da6246bc8 17015 lru 00000080:00200000:0.0:1787826337.138580:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000004ae548aa 17015: 1< 0 - - > 2< 69693440 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138582:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000053668cc1 00000080:00200000:0.0:1787826337.138583:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 120. 00000080:00200000:0.0:1787826337.138585:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17016 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138599:0:68279:0:(rw26.c:875:ll_write_end()) pos 69697536, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138603:0:68279:0:(rw26.c:933:ll_write_end()) page@000000002ca65d01[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17016] 00000080:00200000:0.0:1787826337.138640:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000054e3e727 17ffffc0001001 3:0 ffff907da6246cb0 17016 lru 00000080:00200000:0.0:1787826337.138645:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000ddad5343 17016: 1< 0 - - > 2< 69697536 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138648:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000002ca65d01 00000080:00200000:0.0:1787826337.138649:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 121. 00000080:00200000:0.0:1787826337.138651:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17017 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138667:0:68279:0:(rw26.c:875:ll_write_end()) pos 69701632, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138671:0:68279:0:(rw26.c:933:ll_write_end()) page@000000007baa33b1[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17017] 00000080:00200000:0.0:1787826337.138674:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000bf3c94e6 17ffffc0001001 3:0 ffff907da62469f8 17017 lru 00000080:00200000:0.0:1787826337.138678:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000009c3b4a01 17017: 1< 0 - - > 2< 69701632 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138681:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000007baa33b1 00000080:00200000:0.0:1787826337.138682:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 122. 00000080:00200000:0.0:1787826337.138684:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17018 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138695:0:68279:0:(rw26.c:875:ll_write_end()) pos 69705728, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138699:0:68279:0:(rw26.c:933:ll_write_end()) page@000000009e3023b7[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17018] 00000080:00200000:0.0:1787826337.138701:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000f3a4fc11 17ffffc0001001 3:0 ffff907da6246570 17018 lru 00000080:00200000:0.0:1787826337.138707:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000009c7ce4a7 17018: 1< 0 - - > 2< 69705728 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138709:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000009e3023b7 00000080:00200000:0.0:1787826337.138710:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 123. 00000080:00200000:0.0:1787826337.138712:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17019 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138725:0:68279:0:(rw26.c:875:ll_write_end()) pos 69709824, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138729:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000f6d98115[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17019] 00000080:00200000:0.0:1787826337.138732:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000969e4076 17ffffc0001001 3:0 ffff907da6246828 17019 lru 00000080:00200000:0.0:1787826337.138737:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000005b7cc7ac 17019: 1< 0 - - > 2< 69709824 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138739:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000f6d98115 00000080:00200000:0.0:1787826337.138740:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 124. 00000080:00200000:0.0:1787826337.138742:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17020 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138754:0:68279:0:(rw26.c:875:ll_write_end()) pos 69713920, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138757:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000050607142[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17020] 00000080:00200000:0.0:1787826337.138760:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000361c8ba4 17ffffc0001001 3:0 ffff907da6246d98 17020 lru 00000080:00200000:0.0:1787826337.138765:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000cebdda41 17020: 1< 0 - - > 2< 69713920 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138767:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000050607142 00000080:00200000:0.0:1787826337.138769:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 125. 00000080:00200000:0.0:1787826337.138770:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17021 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138782:0:68279:0:(rw26.c:875:ll_write_end()) pos 69718016, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138786:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000ef4a3b2a[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17021] 00000080:00200000:0.0:1787826337.138789:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000028f10f0e 17ffffc0001001 3:0 ffff907da6246658 17021 lru 00000080:00200000:0.0:1787826337.138793:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000076282d3f 17021: 1< 0 - - > 2< 69718016 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138795:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000ef4a3b2a 00000080:00200000:0.0:1787826337.138795:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 126. 00000080:00200000:0.0:1787826337.138797:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17022 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138809:0:68279:0:(rw26.c:875:ll_write_end()) pos 69722112, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138813:0:68279:0:(rw26.c:933:ll_write_end()) page@000000008efe650b[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17022] 00000080:00200000:0.0:1787826337.138815:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000155fa769 17ffffc0001001 3:0 ffff907da62460e8 17022 lru 00000080:00200000:0.0:1787826337.138820:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000a183473e 17022: 1< 0 - - > 2< 69722112 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138822:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000008efe650b 00000080:00200000:0.0:1787826337.138823:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 127. 00000080:00200000:0.0:1787826337.138825:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17023 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138835:0:68279:0:(rw26.c:875:ll_write_end()) pos 69726208, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138839:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000f23b130e[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17023] 00000080:00200000:0.0:1787826337.138842:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000003941ef99 17ffffc0001001 3:0 ffff907da6246910 17023 lru 00000080:00200000:0.0:1787826337.138846:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000a306bc4f 17023: 1< 0 - - > 2< 69726208 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138849:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000f23b130e 00000080:00200000:0.0:1787826337.138850:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 128. 00000080:00200000:0.0:1787826337.138851:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17024 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138868:0:68279:0:(rw26.c:875:ll_write_end()) pos 69730304, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138871:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000048eeae56[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17024] 00000080:00200000:0.0:1787826337.138874:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000006ab6753e 17ffffc0001001 3:0 ffff907da62461d0 17024 lru 00000080:00200000:0.0:1787826337.138879:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000061e8d2b4 17024: 1< 0 - - > 2< 69730304 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138881:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000048eeae56 00000080:00200000:0.0:1787826337.138882:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 129. 00000080:00200000:0.0:1787826337.138885:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17025 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138900:0:68279:0:(rw26.c:875:ll_write_end()) pos 69734400, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138904:0:68279:0:(rw26.c:933:ll_write_end()) page@000000007a3e4916[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17025] 00000080:00200000:0.0:1787826337.138907:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000d30d1cc6 17ffffc0001021 2:0 ffff907da62462b8 17025 lru 00000080:00200000:0.0:1787826337.138912:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000a906c934 17025: 1< 0 - - > 2< 69734400 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138914:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000007a3e4916 00000080:00200000:0.0:1787826337.138916:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 130. 00000080:00200000:0.0:1787826337.138918:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17026 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138931:0:68279:0:(rw26.c:875:ll_write_end()) pos 69738496, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138936:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000087c0b200[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17026] 00000080:00200000:0.0:1787826337.138938:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000d3277e23 17ffffc0001001 3:0 ffff907da6246000 17026 lru 00000080:00200000:0.0:1787826337.138943:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000e9ee34e5 17026: 1< 0 - - > 2< 69738496 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138945:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000087c0b200 00000080:00200000:0.0:1787826337.138946:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 131. 00000080:00200000:0.0:1787826337.138948:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17027 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138960:0:68279:0:(rw26.c:875:ll_write_end()) pos 69742592, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138965:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000f9a068d5[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17027] 00000080:00200000:0.0:1787826337.138968:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000dc6a9625 17ffffc0001001 3:0 ffff907da6246ae0 17027 lru 00000080:00200000:0.0:1787826337.138973:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000008da915a2 17027: 1< 0 - - > 2< 69742592 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.138976:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000f9a068d5 00000080:00200000:0.0:1787826337.138977:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 132. 00000080:00200000:0.0:1787826337.138978:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17028 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.138991:0:68279:0:(rw26.c:875:ll_write_end()) pos 69746688, len 4096, copied 4096 00000080:00200000:0.0:1787826337.138995:0:68279:0:(rw26.c:933:ll_write_end()) page@000000002da2f38a[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17028] 00000080:00200000:0.0:1787826337.138997:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000001a47fd97 17ffffc0001001 3:0 ffff907da6246e80 17028 lru 00000080:00200000:0.0:1787826337.139003:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000dc1acad8 17028: 1< 0 - - > 2< 69746688 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139005:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000002da2f38a 00000080:00200000:0.0:1787826337.139006:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 133. 00000080:00200000:0.0:1787826337.139008:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17029 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139020:0:68279:0:(rw26.c:875:ll_write_end()) pos 69750784, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139034:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000012175948[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17029] 00000080:00200000:0.0:1787826337.139037:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000000feabea4 17ffffc0001001 3:0 ffff907da62463a0 17029 lru 00000080:00200000:0.0:1787826337.139042:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000006e1b2829 17029: 1< 0 - - > 2< 69750784 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139044:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000012175948 00000080:00200000:0.0:1787826337.139044:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 134. 00000080:00200000:0.0:1787826337.139047:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17030 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139060:0:68279:0:(rw26.c:875:ll_write_end()) pos 69754880, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139064:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000dbaaa24c[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17030] 00000080:00200000:0.0:1787826337.139066:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000022460dd8 17ffffc0001001 3:0 ffff907da6246488 17030 lru 00000080:00200000:0.0:1787826337.139071:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000001cea18d 17030: 1< 0 - - > 2< 69754880 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139072:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000dbaaa24c 00000080:00200000:0.0:1787826337.139073:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 135. 00000080:00200000:0.0:1787826337.139075:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17031 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139087:0:68279:0:(rw26.c:875:ll_write_end()) pos 69758976, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139091:0:68279:0:(rw26.c:933:ll_write_end()) page@000000005fffcd13[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17031] 00000080:00200000:0.0:1787826337.139093:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000e40a5aee 17ffffc0001001 3:0 ffff907da6246740 17031 lru 00000080:00200000:0.0:1787826337.139098:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000de29e63f 17031: 1< 0 - - > 2< 69758976 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139099:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000005fffcd13 00000080:00200000:0.0:1787826337.139100:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 136. 00000080:00200000:0.0:1787826337.139102:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17032 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139115:0:68279:0:(rw26.c:875:ll_write_end()) pos 69763072, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139119:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000c9ad757a[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17032] 00000080:00200000:0.0:1787826337.139121:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000ae270f43 17ffffc0001001 3:0 ffff907d88ba0488 17032 lru 00000080:00200000:0.0:1787826337.139126:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000b50e5716 17032: 1< 0 - - > 2< 69763072 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139128:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000c9ad757a 00000080:00200000:0.0:1787826337.139129:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 137. 00000080:00200000:0.0:1787826337.139131:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17033 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139144:0:68279:0:(rw26.c:875:ll_write_end()) pos 69767168, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139148:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000070499c86[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17033] 00000080:00200000:0.0:1787826337.139151:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000211295c3 17ffffc0001001 3:0 ffff907d88ba0740 17033 lru 00000080:00200000:0.0:1787826337.139155:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000f11c82c3 17033: 1< 0 - - > 2< 69767168 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139158:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000070499c86 00000080:00200000:0.0:1787826337.139159:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 138. 00000080:00200000:0.0:1787826337.139160:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17034 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139171:0:68279:0:(rw26.c:875:ll_write_end()) pos 69771264, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139175:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000073f40666[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17034] 00000080:00200000:0.0:1787826337.139177:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000006af9a77a 17ffffc0001001 3:0 ffff907d88ba0bc8 17034 lru 00000080:00200000:0.0:1787826337.139181:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000003d5684bd 17034: 1< 0 - - > 2< 69771264 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139183:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000073f40666 00000080:00200000:0.0:1787826337.139184:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 139. 00000080:00200000:0.0:1787826337.139185:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17035 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139194:0:68279:0:(rw26.c:875:ll_write_end()) pos 69775360, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139198:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000100e7595[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17035] 00000080:00200000:0.0:1787826337.139200:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000388c096c 17ffffc0001001 3:0 ffff907d88ba0cb0 17035 lru 00000080:00200000:0.0:1787826337.139204:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000001a955546 17035: 1< 0 - - > 2< 69775360 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139206:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000100e7595 00000080:00200000:0.0:1787826337.139206:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 140. 00000080:00200000:0.0:1787826337.139208:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17036 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139217:0:68279:0:(rw26.c:875:ll_write_end()) pos 69779456, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139219:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000f9cfd072[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17036] 00000080:00200000:0.0:1787826337.139221:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000810d114c 17ffffc0001001 3:0 ffff907d88ba09f8 17036 lru 00000080:00200000:0.0:1787826337.139225:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000aa3fb156 17036: 1< 0 - - > 2< 69779456 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139227:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000f9cfd072 00000080:00200000:0.0:1787826337.139227:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 141. 00000080:00200000:0.0:1787826337.139229:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17037 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139238:0:68279:0:(rw26.c:875:ll_write_end()) pos 69783552, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139241:0:68279:0:(rw26.c:933:ll_write_end()) page@000000003dc28be3[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17037] 00000080:00200000:0.0:1787826337.139244:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000013783d64 17ffffc0001001 3:0 ffff907d88ba0570 17037 lru 00000080:00200000:0.0:1787826337.139248:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000491c75af 17037: 1< 0 - - > 2< 69783552 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139249:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000003dc28be3 00000080:00200000:0.0:1787826337.139250:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 142. 00000080:00200000:0.0:1787826337.139251:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17038 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139265:0:68279:0:(rw26.c:875:ll_write_end()) pos 69787648, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139268:0:68279:0:(rw26.c:933:ll_write_end()) page@000000007430da29[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17038] 00000080:00200000:0.0:1787826337.139270:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000ffc48e5e 17ffffc0001001 3:0 ffff907d88ba0828 17038 lru 00000080:00200000:0.0:1787826337.139274:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000009c5f692d 17038: 1< 0 - - > 2< 69787648 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139276:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000007430da29 00000080:00200000:0.0:1787826337.139276:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 143. 00000080:00200000:0.0:1787826337.139278:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17039 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139291:0:68279:0:(rw26.c:875:ll_write_end()) pos 69791744, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139295:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000c3ffb9ce[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17039] 00000080:00200000:0.0:1787826337.139297:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000de10703c 17ffffc0001001 3:0 ffff907d88ba0d98 17039 lru 00000080:00200000:0.0:1787826337.139301:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000a1521cd6 17039: 1< 0 - - > 2< 69791744 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139304:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000c3ffb9ce 00000080:00200000:0.0:1787826337.139304:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 144. 00000080:00200000:0.0:1787826337.139306:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17040 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139320:0:68279:0:(rw26.c:875:ll_write_end()) pos 69795840, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139324:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000b5ef9739[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17040] 00000080:00200000:0.0:1787826337.139327:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000013daa70d 17ffffc0001021 2:0 ffff907d88ba0658 17040 lru 00000080:00200000:0.0:1787826337.139331:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000051b4457c 17040: 1< 0 - - > 2< 69795840 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139333:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000b5ef9739 00000080:00200000:0.0:1787826337.139333:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 145. 00000080:00200000:0.0:1787826337.139335:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17041 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139346:0:68279:0:(rw26.c:875:ll_write_end()) pos 69799936, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139350:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000e95b318c[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17041] 00000080:00200000:0.0:1787826337.139352:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000922dc2a3 17ffffc0001001 3:0 ffff907d88ba00e8 17041 lru 00000080:00200000:0.0:1787826337.139357:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000887d2898 17041: 1< 0 - - > 2< 69799936 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139359:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000e95b318c 00000080:00200000:0.0:1787826337.139359:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 146. 00000080:00200000:0.0:1787826337.139362:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17042 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139373:0:68279:0:(rw26.c:875:ll_write_end()) pos 69804032, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139376:0:68279:0:(rw26.c:933:ll_write_end()) page@000000000075d576[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17042] 00000080:00200000:0.0:1787826337.139379:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000104b2894 17ffffc0001001 3:0 ffff907d88ba0910 17042 lru 00000080:00200000:0.0:1787826337.139383:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000009093377e 17042: 1< 0 - - > 2< 69804032 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139385:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000000075d576 00000080:00200000:0.0:1787826337.139386:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 147. 00000080:00200000:0.0:1787826337.139387:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17043 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139399:0:68279:0:(rw26.c:875:ll_write_end()) pos 69808128, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139402:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000038ec8628[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17043] 00000080:00200000:0.0:1787826337.139404:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000acbe7a7b 17ffffc0001001 3:0 ffff907d88ba01d0 17043 lru 00000080:00200000:0.0:1787826337.139408:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000006657ce77 17043: 1< 0 - - > 2< 69808128 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139410:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000038ec8628 00000080:00200000:0.0:1787826337.139411:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 148. 00000080:00200000:0.0:1787826337.139413:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17044 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139423:0:68279:0:(rw26.c:875:ll_write_end()) pos 69812224, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139426:0:68279:0:(rw26.c:933:ll_write_end()) page@000000009b1efbf2[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17044] 00000080:00200000:0.0:1787826337.139429:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000d6944145 17ffffc0001001 3:0 ffff907d88ba02b8 17044 lru 00000080:00200000:0.0:1787826337.139434:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000042a5b54b 17044: 1< 0 - - > 2< 69812224 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139436:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000009b1efbf2 00000080:00200000:0.0:1787826337.139437:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 149. 00000080:00200000:0.0:1787826337.139438:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17045 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139452:0:68279:0:(rw26.c:875:ll_write_end()) pos 69816320, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139456:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000b1261361[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17045] 00000080:00200000:0.0:1787826337.139459:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000a657cceb 17ffffc0001001 3:0 ffff907d88ba0000 17045 lru 00000080:00200000:0.0:1787826337.139463:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000003f594a36 17045: 1< 0 - - > 2< 69816320 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139465:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000b1261361 00000080:00200000:0.0:1787826337.139466:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 150. 00000080:00200000:0.0:1787826337.139468:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17046 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139479:0:68279:0:(rw26.c:875:ll_write_end()) pos 69820416, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139484:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000d31237ac[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17046] 00000080:00200000:0.0:1787826337.139486:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000004b2a706f 17ffffc0001001 3:0 ffff907d88ba0ae0 17046 lru 00000080:00200000:0.0:1787826337.139491:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000008f74e864 17046: 1< 0 - - > 2< 69820416 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139493:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000d31237ac 00000080:00200000:0.0:1787826337.139494:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 151. 00000080:00200000:0.0:1787826337.139496:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17047 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139509:0:68279:0:(rw26.c:875:ll_write_end()) pos 69824512, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139512:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000bfd95c46[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17047] 00000080:00200000:0.0:1787826337.139514:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000098e4611a 17ffffc0001001 3:0 ffff907d88ba0e80 17047 lru 00000080:00200000:0.0:1787826337.139519:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000001caba05c 17047: 1< 0 - - > 2< 69824512 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139521:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000bfd95c46 00000080:00200000:0.0:1787826337.139521:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 152. 00000080:00200000:0.0:1787826337.139523:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17048 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139535:0:68279:0:(rw26.c:875:ll_write_end()) pos 69828608, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139538:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000d04e53b7[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17048] 00000080:00200000:0.0:1787826337.139540:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000006a29a5d8 17ffffc0001001 3:0 ffff907d88ba03a0 17048 lru 00000080:00200000:0.0:1787826337.139545:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000006751b483 17048: 1< 0 - - > 2< 69828608 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139547:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000d04e53b7 00000080:00200000:0.0:1787826337.139548:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 153. 00000080:00200000:0.0:1787826337.139549:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17049 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139561:0:68279:0:(rw26.c:875:ll_write_end()) pos 69832704, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139564:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000072867d1a[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17049] 00000080:00200000:0.0:1787826337.139566:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000acfb55e4 17ffffc0001001 3:0 ffff907d9603a2b8 17049 lru 00000080:00200000:0.0:1787826337.139571:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000ac59bd62 17049: 1< 0 - - > 2< 69832704 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139573:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000072867d1a 00000080:00200000:0.0:1787826337.139574:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 154. 00000080:00200000:0.0:1787826337.139576:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17050 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139589:0:68279:0:(rw26.c:875:ll_write_end()) pos 69836800, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139593:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000acf52f54[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17050] 00000080:00200000:0.0:1787826337.139596:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000001ea8483f 17ffffc0001001 3:0 ffff907d9603a000 17050 lru 00000080:00200000:0.0:1787826337.139600:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000dc888ad6 17050: 1< 0 - - > 2< 69836800 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139602:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000acf52f54 00000080:00200000:0.0:1787826337.139603:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 155. 00000080:00200000:0.0:1787826337.139633:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17051 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139648:0:68279:0:(rw26.c:875:ll_write_end()) pos 69840896, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139653:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000ea17e4ea[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17051] 00000080:00200000:0.0:1787826337.139656:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000f00364d3 17ffffc0001001 3:0 ffff907d9603aae0 17051 lru 00000080:00200000:0.0:1787826337.139662:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000a015877d 17051: 1< 0 - - > 2< 69840896 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139664:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000ea17e4ea 00000080:00200000:0.0:1787826337.139666:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 156. 00000080:00200000:0.0:1787826337.139668:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17052 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139681:0:68279:0:(rw26.c:875:ll_write_end()) pos 69844992, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139686:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000a9c81f58[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17052] 00000080:00200000:0.0:1787826337.139689:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000515062d9 17ffffc0001001 3:0 ffff907d9603ae80 17052 lru 00000080:00200000:0.0:1787826337.139694:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000032a9288b 17052: 1< 0 - - > 2< 69844992 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139696:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000a9c81f58 00000080:00200000:0.0:1787826337.139697:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 157. 00000080:00200000:0.0:1787826337.139700:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17053 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139712:0:68279:0:(rw26.c:875:ll_write_end()) pos 69849088, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139717:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000dfce20a4[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17053] 00000080:00200000:0.0:1787826337.139720:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000c81cd894 17ffffc0001001 3:0 ffff907d9603a3a0 17053 lru 00000080:00200000:0.0:1787826337.139725:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000e09cd130 17053: 1< 0 - - > 2< 69849088 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139727:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000dfce20a4 00000080:00200000:0.0:1787826337.139729:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 158. 00000080:00200000:0.0:1787826337.139731:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17054 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139744:0:68279:0:(rw26.c:875:ll_write_end()) pos 69853184, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139748:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000478d4658[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17054] 00000080:00200000:0.0:1787826337.139751:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000026f8c73b 17ffffc0001001 3:0 ffff907d9603a488 17054 lru 00000080:00200000:0.0:1787826337.139757:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000cf3df37c 17054: 1< 0 - - > 2< 69853184 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139759:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000478d4658 00000080:00200000:0.0:1787826337.139760:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 159. 00000080:00200000:0.0:1787826337.139762:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17055 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139777:0:68279:0:(rw26.c:875:ll_write_end()) pos 69857280, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139782:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000053787f24[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17055] 00000080:00200000:0.0:1787826337.139784:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000cd574f04 17ffffc0001021 2:0 ffff907d9603a740 17055 lru 00000080:00200000:0.0:1787826337.139789:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000009e154933 17055: 1< 0 - - > 2< 69857280 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139792:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000053787f24 00000080:00200000:0.0:1787826337.139792:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 160. 00000080:00200000:0.0:1787826337.139794:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17056 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139809:0:68279:0:(rw26.c:875:ll_write_end()) pos 69861376, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139813:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000cc3e90f8[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17056] 00000080:00200000:0.0:1787826337.139815:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000081f1f320 17ffffc0001001 3:0 ffff907d9603abc8 17056 lru 00000080:00200000:0.0:1787826337.139820:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000018682c26 17056: 1< 0 - - > 2< 69861376 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139821:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000cc3e90f8 00000080:00200000:0.0:1787826337.139822:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 161. 00000080:00200000:0.0:1787826337.139824:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17057 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139837:0:68279:0:(rw26.c:875:ll_write_end()) pos 69865472, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139840:0:68279:0:(rw26.c:933:ll_write_end()) page@000000003b81f373[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17057] 00000080:00200000:0.0:1787826337.139843:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000abd1dd57 17ffffc0001001 3:0 ffff907d9603acb0 17057 lru 00000080:00200000:0.0:1787826337.139848:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000042cec93d 17057: 1< 0 - - > 2< 69865472 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139850:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000003b81f373 00000080:00200000:0.0:1787826337.139851:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 162. 00000080:00200000:0.0:1787826337.139853:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17058 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139866:0:68279:0:(rw26.c:875:ll_write_end()) pos 69869568, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139871:0:68279:0:(rw26.c:933:ll_write_end()) page@000000009a4e15a9[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17058] 00000080:00200000:0.0:1787826337.139873:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000949e3206 17ffffc0001001 3:0 ffff907d9603a9f8 17058 lru 00000080:00200000:0.0:1787826337.139878:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000005f842ed1 17058: 1< 0 - - > 2< 69869568 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139880:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000009a4e15a9 00000080:00200000:0.0:1787826337.139881:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 163. 00000080:00200000:0.0:1787826337.139883:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17059 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139899:0:68279:0:(rw26.c:875:ll_write_end()) pos 69873664, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139903:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000c1a7fc24[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17059] 00000080:00200000:0.0:1787826337.139906:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000647c68e7 17ffffc0001001 3:0 ffff907d9603a570 17059 lru 00000080:00200000:0.0:1787826337.139910:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000736d8d5b 17059: 1< 0 - - > 2< 69873664 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139913:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000c1a7fc24 00000080:00200000:0.0:1787826337.139914:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 164. 00000080:00200000:0.0:1787826337.139916:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17060 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139929:0:68279:0:(rw26.c:875:ll_write_end()) pos 69877760, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139933:0:68279:0:(rw26.c:933:ll_write_end()) page@000000002fc8c726[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17060] 00000080:00200000:0.0:1787826337.139936:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000bc0ce7eb 17ffffc0001001 3:0 ffff907d9603a828 17060 lru 00000080:00200000:0.0:1787826337.139941:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000eb9ac184 17060: 1< 0 - - > 2< 69877760 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.139943:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000002fc8c726 00000080:00200000:0.0:1787826337.139944:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 165. 00000080:00200000:0.0:1787826337.139945:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17061 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.139958:0:68279:0:(rw26.c:875:ll_write_end()) pos 69881856, len 4096, copied 4096 00000080:00200000:0.0:1787826337.139962:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000a50dc127[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17061] 00000080:00200000:0.0:1787826337.142526:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000040ec6e37 17ffffc0001001 3:0 ffff907d9603ad98 17061 lru 00000080:00200000:0.0:1787826337.142541:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000030952619 17061: 1< 0 - - > 2< 69881856 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.142544:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000a50dc127 00000080:00200000:0.0:1787826337.142545:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 166. 00000080:00200000:0.0:1787826337.142552:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17062 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.142596:0:68279:0:(rw26.c:875:ll_write_end()) pos 69885952, len 4096, copied 4096 00000080:00200000:0.0:1787826337.142603:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000f32ba233[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17062] 00000080:00200000:0.0:1787826337.142631:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000009217d36c 17ffffc0001001 3:0 ffff907d9603a658 17062 lru 00000080:00200000:0.0:1787826337.142637:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000ccf6a0af 17062: 1< 0 - - > 2< 69885952 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.142639:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000f32ba233 00000080:00200000:0.0:1787826337.142641:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 167. 00000080:00200000:0.0:1787826337.142643:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17063 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.142659:0:68279:0:(rw26.c:875:ll_write_end()) pos 69890048, len 4096, copied 4096 00000080:00200000:0.0:1787826337.142663:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000f4dcc8cd[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17063] 00000080:00200000:0.0:1787826337.142666:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000df96da73 17ffffc0001001 3:0 ffff907d9603a0e8 17063 lru 00000080:00200000:0.0:1787826337.142671:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000e9bfe061 17063: 1< 0 - - > 2< 69890048 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.142673:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000f4dcc8cd 00000080:00200000:0.0:1787826337.142674:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 168. 00000080:00200000:0.0:1787826337.142676:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17064 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.142690:0:68279:0:(rw26.c:875:ll_write_end()) pos 69894144, len 4096, copied 4096 00000080:00200000:0.0:1787826337.142695:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000758a1136[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17064] 00000080:00200000:0.0:1787826337.142697:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000a6e7f7d4 17ffffc0001001 3:0 ffff907d9603a910 17064 lru 00000080:00200000:0.0:1787826337.142702:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000e924b7b1 17064: 1< 0 - - > 2< 69894144 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.142704:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000758a1136 00000080:00200000:0.0:1787826337.142705:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 169. 00000080:00200000:0.0:1787826337.142707:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17065 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.142718:0:68279:0:(rw26.c:875:ll_write_end()) pos 69898240, len 4096, copied 4096 00000080:00200000:0.0:1787826337.142722:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000885b8663[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17065] 00000080:00200000:0.0:1787826337.142725:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000006af1cb 17ffffc0001001 3:0 ffff907d9603a1d0 17065 lru 00000080:00200000:0.0:1787826337.142730:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000006c09b5f9 17065: 1< 0 - - > 2< 69898240 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.142732:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000885b8663 00000080:00200000:0.0:1787826337.142732:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 170. 00000080:00200000:0.0:1787826337.142734:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17066 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.142750:0:68279:0:(rw26.c:875:ll_write_end()) pos 69902336, len 4096, copied 4096 00000080:00200000:0.0:1787826337.142754:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000800aecfc[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17066] 00000080:00200000:0.0:1787826337.142757:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000002bf6291d 17ffffc0001001 3:0 ffff907d83406ae0 17066 lru 00000080:00200000:0.0:1787826337.142761:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000002142a0f 17066: 1< 0 - - > 2< 69902336 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.142763:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000800aecfc 00000080:00200000:0.0:1787826337.142764:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 171. 00000080:00200000:0.0:1787826337.142766:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17067 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.142779:0:68279:0:(rw26.c:875:ll_write_end()) pos 69906432, len 4096, copied 4096 00000080:00200000:0.0:1787826337.142782:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000f513b7b2[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17067] 00000080:00200000:0.0:1787826337.142785:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000eb980f61 17ffffc0001001 3:0 ffff907d83406e80 17067 lru 00000080:00200000:0.0:1787826337.142790:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000de2b784d 17067: 1< 0 - - > 2< 69906432 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.142792:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000f513b7b2 00000080:00200000:0.0:1787826337.142793:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 172. 00000080:00200000:0.0:1787826337.142795:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17068 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.142808:0:68279:0:(rw26.c:875:ll_write_end()) pos 69910528, len 4096, copied 4096 00000080:00200000:0.0:1787826337.142812:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000440bbf02[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17068] 00000080:00200000:0.0:1787826337.142815:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000b94f0417 17ffffc0001001 3:0 ffff907d834063a0 17068 lru 00000080:00200000:0.0:1787826337.142819:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000053b3329c 17068: 1< 0 - - > 2< 69910528 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.142821:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000440bbf02 00000080:00200000:0.0:1787826337.142822:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 173. 00000080:00200000:0.0:1787826337.142824:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17069 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.142836:0:68279:0:(rw26.c:875:ll_write_end()) pos 69914624, len 4096, copied 4096 00000080:00200000:0.0:1787826337.142840:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000de0f517b[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17069] 00000080:00200000:0.0:1787826337.142842:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000095c1d57b 17ffffc0001001 3:0 ffff907d83406488 17069 lru 00000080:00200000:0.0:1787826337.142847:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000644b9d46 17069: 1< 0 - - > 2< 69914624 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.142848:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000de0f517b 00000080:00200000:0.0:1787826337.142850:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 174. 00000080:00200000:0.0:1787826337.142851:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17070 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.142871:0:68279:0:(rw26.c:875:ll_write_end()) pos 69918720, len 4096, copied 4096 00000080:00200000:0.0:1787826337.142875:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000084f567e1[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17070] 00000080:00200000:0.0:1787826337.142878:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000050fda78e 17ffffc0001021 2:0 ffff907d83406740 17070 lru 00000080:00200000:0.0:1787826337.142883:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000000e855ba8 17070: 1< 0 - - > 2< 69918720 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.142885:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000084f567e1 00000080:00200000:0.0:1787826337.142886:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 175. 00000080:00200000:0.0:1787826337.142887:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17071 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.142900:0:68279:0:(rw26.c:875:ll_write_end()) pos 69922816, len 4096, copied 4096 00000080:00200000:0.0:1787826337.142904:0:68279:0:(rw26.c:933:ll_write_end()) page@000000009e7a33be[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17071] 00000080:00200000:0.0:1787826337.142907:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000ab59c02c 17ffffc0001001 3:0 ffff907d83406bc8 17071 lru 00000080:00200000:0.0:1787826337.142912:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000eb9f1f53 17071: 1< 0 - - > 2< 69922816 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.142914:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000009e7a33be 00000080:00200000:0.0:1787826337.142915:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 176. 00000080:00200000:0.0:1787826337.142918:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17072 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.142931:0:68279:0:(rw26.c:875:ll_write_end()) pos 69926912, len 4096, copied 4096 00000080:00200000:0.0:1787826337.142935:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000054aa966a[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17072] 00000080:00200000:0.0:1787826337.142938:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000a5e5fbd0 17ffffc0001001 3:0 ffff907d83406cb0 17072 lru 00000080:00200000:0.0:1787826337.142942:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000dc2d1aa0 17072: 1< 0 - - > 2< 69926912 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.142944:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000054aa966a 00000080:00200000:0.0:1787826337.142945:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 177. 00000080:00200000:0.0:1787826337.142947:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17073 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.142961:0:68279:0:(rw26.c:875:ll_write_end()) pos 69931008, len 4096, copied 4096 00000080:00200000:0.0:1787826337.142965:0:68279:0:(rw26.c:933:ll_write_end()) page@000000008987d4e9[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17073] 00000080:00200000:0.0:1787826337.142968:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000ca833d5f 17ffffc0001001 3:0 ffff907d834069f8 17073 lru 00000080:00200000:0.0:1787826337.142973:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000cbe4d12a 17073: 1< 0 - - > 2< 69931008 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.142976:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000008987d4e9 00000080:00200000:0.0:1787826337.142977:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 178. 00000080:00200000:0.0:1787826337.142978:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17074 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.142992:0:68279:0:(rw26.c:875:ll_write_end()) pos 69935104, len 4096, copied 4096 00000080:00200000:0.0:1787826337.142996:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000d665f023[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17074] 00000080:00200000:0.0:1787826337.142999:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000007cf36925 17ffffc0001001 3:0 ffff907d83406570 17074 lru 00000080:00200000:0.0:1787826337.143004:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000d2cb436d 17074: 1< 0 - - > 2< 69935104 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143006:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000d665f023 00000080:00200000:0.0:1787826337.143007:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 179. 00000080:00200000:0.0:1787826337.143009:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17075 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143021:0:68279:0:(rw26.c:875:ll_write_end()) pos 69939200, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143037:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000041a4dca7[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17075] 00000080:00200000:0.0:1787826337.143040:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000083fa0528 17ffffc0001001 3:0 ffff907d83406828 17075 lru 00000080:00200000:0.0:1787826337.143045:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000d12d873d 17075: 1< 0 - - > 2< 69939200 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143047:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000041a4dca7 00000080:00200000:0.0:1787826337.143048:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 180. 00000080:00200000:0.0:1787826337.143050:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17076 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143064:0:68279:0:(rw26.c:875:ll_write_end()) pos 69943296, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143068:0:68279:0:(rw26.c:933:ll_write_end()) page@000000006353f67f[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17076] 00000080:00200000:0.0:1787826337.143070:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000b99e0b5c 17ffffc0001001 3:0 ffff907d83406d98 17076 lru 00000080:00200000:0.0:1787826337.143074:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000f46addd1 17076: 1< 0 - - > 2< 69943296 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143076:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000006353f67f 00000080:00200000:0.0:1787826337.143077:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 181. 00000080:00200000:0.0:1787826337.143079:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17077 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143090:0:68279:0:(rw26.c:875:ll_write_end()) pos 69947392, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143093:0:68279:0:(rw26.c:933:ll_write_end()) page@000000003eebd60c[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17077] 00000080:00200000:0.0:1787826337.143096:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000008f43a6c5 17ffffc0001001 3:0 ffff907d83406658 17077 lru 00000080:00200000:0.0:1787826337.143100:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000002d49de08 17077: 1< 0 - - > 2< 69947392 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143101:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000003eebd60c 00000080:00200000:0.0:1787826337.143102:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 182. 00000080:00200000:0.0:1787826337.143103:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17078 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143114:0:68279:0:(rw26.c:875:ll_write_end()) pos 69951488, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143117:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000026210146[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17078] 00000080:00200000:0.0:1787826337.143119:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000877992cf 17ffffc0001001 3:0 ffff907d834060e8 17078 lru 00000080:00200000:0.0:1787826337.143124:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000e24abeba 17078: 1< 0 - - > 2< 69951488 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143126:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000026210146 00000080:00200000:0.0:1787826337.143127:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 183. 00000080:00200000:0.0:1787826337.143128:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17079 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143140:0:68279:0:(rw26.c:875:ll_write_end()) pos 69955584, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143144:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000584b1933[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17079] 00000080:00200000:0.0:1787826337.143147:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000b36b23ec 17ffffc0001001 3:0 ffff907d83406910 17079 lru 00000080:00200000:0.0:1787826337.143151:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000f6ab3b5b 17079: 1< 0 - - > 2< 69955584 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143154:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000584b1933 00000080:00200000:0.0:1787826337.143155:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 184. 00000080:00200000:0.0:1787826337.143157:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17080 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143171:0:68279:0:(rw26.c:875:ll_write_end()) pos 69959680, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143175:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000025529e28[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17080] 00000080:00200000:0.0:1787826337.143178:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000c38fa37b 17ffffc0001001 3:0 ffff907d834061d0 17080 lru 00000080:00200000:0.0:1787826337.143183:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000cabe482b 17080: 1< 0 - - > 2< 69959680 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143185:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000025529e28 00000080:00200000:0.0:1787826337.143186:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 185. 00000080:00200000:0.0:1787826337.143189:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17081 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143201:0:68279:0:(rw26.c:875:ll_write_end()) pos 69963776, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143205:0:68279:0:(rw26.c:933:ll_write_end()) page@000000002f7c706d[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17081] 00000080:00200000:0.0:1787826337.143207:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000004162d05a 17ffffc0001001 3:0 ffff907d834062b8 17081 lru 00000080:00200000:0.0:1787826337.143213:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000008fccbbfa 17081: 1< 0 - - > 2< 69963776 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143215:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000002f7c706d 00000080:00200000:0.0:1787826337.143216:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 186. 00000080:00200000:0.0:1787826337.143217:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17082 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143230:0:68279:0:(rw26.c:875:ll_write_end()) pos 69967872, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143234:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000728245d8[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17082] 00000080:00200000:0.0:1787826337.143237:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000069793820 17ffffc0001001 3:0 ffff907d83406000 17082 lru 00000080:00200000:0.0:1787826337.143242:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000009ce94504 17082: 1< 0 - - > 2< 69967872 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143244:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000728245d8 00000080:00200000:0.0:1787826337.143245:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 187. 00000080:00200000:0.0:1787826337.143247:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17083 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143262:0:68279:0:(rw26.c:875:ll_write_end()) pos 69971968, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143266:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000892a8c72[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17083] 00000080:00200000:0.0:1787826337.143268:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000076f55304 17ffffc0001001 3:0 ffff907d9355f828 17083 lru 00000080:00200000:0.0:1787826337.143273:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000007b514ff0 17083: 1< 0 - - > 2< 69971968 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143275:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000892a8c72 00000080:00200000:0.0:1787826337.143276:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 188. 00000080:00200000:0.0:1787826337.143277:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17084 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143604:0:68279:0:(rw26.c:875:ll_write_end()) pos 69976064, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143636:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000b5f4de3b[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17084] 00000080:00200000:0.0:1787826337.143639:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000948f18ba 17ffffc0001001 3:0 ffff907d9355fd98 17084 lru 00000080:00200000:0.0:1787826337.143644:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000b08cf3a7 17084: 1< 0 - - > 2< 69976064 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143646:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000b5f4de3b 00000080:00200000:0.0:1787826337.143648:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 189. 00000080:00200000:0.0:1787826337.143650:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17085 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143669:0:68279:0:(rw26.c:875:ll_write_end()) pos 69980160, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143674:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000fed8beec[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17085] 00000080:00200000:0.0:1787826337.143676:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000e0661e26 17ffffc0001021 2:0 ffff907d9355f658 17085 lru 00000080:00200000:0.0:1787826337.143681:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000063632292 17085: 1< 0 - - > 2< 69980160 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143683:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000fed8beec 00000080:00200000:0.0:1787826337.143684:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 190. 00000080:00200000:0.0:1787826337.143687:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17086 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143701:0:68279:0:(rw26.c:875:ll_write_end()) pos 69984256, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143706:0:68279:0:(rw26.c:933:ll_write_end()) page@000000006e28b5bf[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17086] 00000080:00200000:0.0:1787826337.143709:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000cbac795d 17ffffc0001001 3:0 ffff907d9355f0e8 17086 lru 00000080:00200000:0.0:1787826337.143714:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000074101da4 17086: 1< 0 - - > 2< 69984256 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143716:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000006e28b5bf 00000080:00200000:0.0:1787826337.143717:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 191. 00000080:00200000:0.0:1787826337.143720:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17087 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143736:0:68279:0:(rw26.c:875:ll_write_end()) pos 69988352, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143741:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000ce9b3207[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17087] 00000080:00200000:0.0:1787826337.143743:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000009bada09c 17ffffc0001001 3:0 ffff907d9355f910 17087 lru 00000080:00200000:0.0:1787826337.143749:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000c8b80a6c 17087: 1< 0 - - > 2< 69988352 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143751:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000ce9b3207 00000080:00200000:0.0:1787826337.143752:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 192. 00000080:00200000:0.0:1787826337.143754:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17088 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143773:0:68279:0:(rw26.c:875:ll_write_end()) pos 69992448, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143778:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000aa0c8f0b[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17088] 00000080:00200000:0.0:1787826337.143781:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000098b69508 17ffffc0001001 3:0 ffff907d9355f1d0 17088 lru 00000080:00200000:0.0:1787826337.143786:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000003f752555 17088: 1< 0 - - > 2< 69992448 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143788:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000aa0c8f0b 00000080:00200000:0.0:1787826337.143789:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 193. 00000080:00200000:0.0:1787826337.143791:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17089 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143807:0:68279:0:(rw26.c:875:ll_write_end()) pos 69996544, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143812:0:68279:0:(rw26.c:933:ll_write_end()) page@000000003c3d7060[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17089] 00000080:00200000:0.0:1787826337.143815:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000a6953499 17ffffc0001001 3:0 ffff907d9355f2b8 17089 lru 00000080:00200000:0.0:1787826337.143819:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000f427ed22 17089: 1< 0 - - > 2< 69996544 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143821:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000003c3d7060 00000080:00200000:0.0:1787826337.143822:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 194. 00000080:00200000:0.0:1787826337.143824:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17090 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143839:0:68279:0:(rw26.c:875:ll_write_end()) pos 70000640, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143843:0:68279:0:(rw26.c:933:ll_write_end()) page@000000008fff12c7[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17090] 00000080:00200000:0.0:1787826337.143846:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000002a676ece 17ffffc0001001 3:0 ffff907d9355f000 17090 lru 00000080:00200000:0.0:1787826337.143851:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000067754d5f 17090: 1< 0 - - > 2< 70000640 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143853:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000008fff12c7 00000080:00200000:0.0:1787826337.143854:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 195. 00000080:00200000:0.0:1787826337.143856:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17091 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143868:0:68279:0:(rw26.c:875:ll_write_end()) pos 70004736, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143872:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000025630544[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17091] 00000080:00200000:0.0:1787826337.143875:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000001699e675 17ffffc0001001 3:0 ffff907d9355fae0 17091 lru 00000080:00200000:0.0:1787826337.143879:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000075a8d8d9 17091: 1< 0 - - > 2< 70004736 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143882:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000025630544 00000080:00200000:0.0:1787826337.143883:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 196. 00000080:00200000:0.0:1787826337.143885:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17092 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143897:0:68279:0:(rw26.c:875:ll_write_end()) pos 70008832, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143901:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000c58b891c[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17092] 00000080:00200000:0.0:1787826337.143903:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000002ae1a7a4 17ffffc0001001 3:0 ffff907d9355fe80 17092 lru 00000080:00200000:0.0:1787826337.143908:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000fa24dfa7 17092: 1< 0 - - > 2< 70008832 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143911:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000c58b891c 00000080:00200000:0.0:1787826337.143912:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 197. 00000080:00200000:0.0:1787826337.143913:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17093 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143926:0:68279:0:(rw26.c:875:ll_write_end()) pos 70012928, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143930:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000eb5a004f[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17093] 00000080:00200000:0.0:1787826337.143933:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000841d3a69 17ffffc0001001 3:0 ffff907d9355f3a0 17093 lru 00000080:00200000:0.0:1787826337.143938:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000007b7710e1 17093: 1< 0 - - > 2< 70012928 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143940:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000eb5a004f 00000080:00200000:0.0:1787826337.143941:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 198. 00000080:00200000:0.0:1787826337.143942:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17094 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143957:0:68279:0:(rw26.c:875:ll_write_end()) pos 70017024, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143961:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000153bb963[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17094] 00000080:00200000:0.0:1787826337.143964:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000a429bb38 17ffffc0001001 3:0 ffff907d9355f488 17094 lru 00000080:00200000:0.0:1787826337.143968:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000c056bfb0 17094: 1< 0 - - > 2< 70017024 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143970:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000153bb963 00000080:00200000:0.0:1787826337.143971:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 199. 00000080:00200000:0.0:1787826337.143973:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17095 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.143986:0:68279:0:(rw26.c:875:ll_write_end()) pos 70021120, len 4096, copied 4096 00000080:00200000:0.0:1787826337.143990:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000e0320836[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17095] 00000080:00200000:0.0:1787826337.143992:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000009a3960e0 17ffffc0001001 3:0 ffff907d9355f740 17095 lru 00000080:00200000:0.0:1787826337.143997:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000055f365a0 17095: 1< 0 - - > 2< 70021120 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.143999:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000e0320836 00000080:00200000:0.0:1787826337.144001:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 200. 00000080:00200000:0.0:1787826337.144002:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17096 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144016:0:68279:0:(rw26.c:875:ll_write_end()) pos 70025216, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144020:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000be37a676[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17096] 00000080:00200000:0.0:1787826337.144022:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000f7ce17e7 17ffffc0001001 3:0 ffff907d9355fbc8 17096 lru 00000080:00200000:0.0:1787826337.144039:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000007878656a 17096: 1< 0 - - > 2< 70025216 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144041:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000be37a676 00000080:00200000:0.0:1787826337.144043:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 201. 00000080:00200000:0.0:1787826337.144045:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17097 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144059:0:68279:0:(rw26.c:875:ll_write_end()) pos 70029312, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144063:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000bac36094[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17097] 00000080:00200000:0.0:1787826337.144066:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000002f19080d 17ffffc0001001 3:0 ffff907d9355fcb0 17097 lru 00000080:00200000:0.0:1787826337.144070:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000042a04f0d 17097: 1< 0 - - > 2< 70029312 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144072:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000bac36094 00000080:00200000:0.0:1787826337.144073:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 202. 00000080:00200000:0.0:1787826337.144075:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17098 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144087:0:68279:0:(rw26.c:875:ll_write_end()) pos 70033408, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144090:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000d9ab9a4c[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17098] 00000080:00200000:0.0:1787826337.144093:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000009d7f6c9 17ffffc0001001 3:0 ffff907d9355f9f8 17098 lru 00000080:00200000:0.0:1787826337.144098:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000032ef9551 17098: 1< 0 - - > 2< 70033408 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144100:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000d9ab9a4c 00000080:00200000:0.0:1787826337.144101:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 203. 00000080:00200000:0.0:1787826337.144103:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17099 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144115:0:68279:0:(rw26.c:875:ll_write_end()) pos 70037504, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144119:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000b395b4ad[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17099] 00000080:00200000:0.0:1787826337.144122:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000071b01f97 17ffffc0001001 3:0 ffff907d9355f570 17099 lru 00000080:00200000:0.0:1787826337.144127:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000057dac6e4 17099: 1< 0 - - > 2< 70037504 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144129:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000b395b4ad 00000080:00200000:0.0:1787826337.144130:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 204. 00000080:00200000:0.0:1787826337.144132:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17100 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144151:0:68279:0:(rw26.c:875:ll_write_end()) pos 70041600, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144155:0:68279:0:(rw26.c:933:ll_write_end()) page@000000003604ce68[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17100] 00000080:00200000:0.0:1787826337.144158:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000c8d4060e 17ffffc0001021 2:0 ffff907d9170de80 17100 lru 00000080:00200000:0.0:1787826337.144163:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000962df693 17100: 1< 0 - - > 2< 70041600 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144165:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000003604ce68 00000080:00200000:0.0:1787826337.144166:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 205. 00000080:00200000:0.0:1787826337.144167:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17101 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144180:0:68279:0:(rw26.c:875:ll_write_end()) pos 70045696, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144185:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000ce42eb6e[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17101] 00000080:00200000:0.0:1787826337.144187:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000b26cd299 17ffffc0001001 3:0 ffff907d9170d3a0 17101 lru 00000080:00200000:0.0:1787826337.144192:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000bca64ca2 17101: 1< 0 - - > 2< 70045696 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144194:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000ce42eb6e 00000080:00200000:0.0:1787826337.144195:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 206. 00000080:00200000:0.0:1787826337.144197:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17102 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144213:0:68279:0:(rw26.c:875:ll_write_end()) pos 70049792, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144218:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000095322a52[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17102] 00000080:00200000:0.0:1787826337.144220:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000eb71016c 17ffffc0001001 3:0 ffff907d9170d488 17102 lru 00000080:00200000:0.0:1787826337.144225:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000ea019d4a 17102: 1< 0 - - > 2< 70049792 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144228:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000095322a52 00000080:00200000:0.0:1787826337.144229:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 207. 00000080:00200000:0.0:1787826337.144231:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17103 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144244:0:68279:0:(rw26.c:875:ll_write_end()) pos 70053888, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144248:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000f6adfcaf[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17103] 00000080:00200000:0.0:1787826337.144251:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000004375065b 17ffffc0001001 3:0 ffff907d9170d740 17103 lru 00000080:00200000:0.0:1787826337.144257:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000ddd7ed76 17103: 1< 0 - - > 2< 70053888 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144259:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000f6adfcaf 00000080:00200000:0.0:1787826337.144260:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 208. 00000080:00200000:0.0:1787826337.144261:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17104 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144275:0:68279:0:(rw26.c:875:ll_write_end()) pos 70057984, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144279:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000bc494e35[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17104] 00000080:00200000:0.0:1787826337.144281:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000708c914a 17ffffc0001001 3:0 ffff907d9170dbc8 17104 lru 00000080:00200000:0.0:1787826337.144286:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000f7509509 17104: 1< 0 - - > 2< 70057984 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144552:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000bc494e35 00000080:00200000:0.0:1787826337.144553:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 209. 00000080:00200000:0.0:1787826337.144556:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17105 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144572:0:68279:0:(rw26.c:875:ll_write_end()) pos 70062080, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144577:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000083af8e90[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17105] 00000080:00200000:0.0:1787826337.144580:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000a8ded591 17ffffc0001001 3:0 ffff907d9170dcb0 17105 lru 00000080:00200000:0.0:1787826337.144585:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000003b9742a6 17105: 1< 0 - - > 2< 70062080 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144587:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000083af8e90 00000080:00200000:0.0:1787826337.144588:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 210. 00000080:00200000:0.0:1787826337.144591:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17106 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144604:0:68279:0:(rw26.c:875:ll_write_end()) pos 70066176, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144632:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000dfc88ae4[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17106] 00000080:00200000:0.0:1787826337.144634:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000008d56f9d8 17ffffc0001001 3:0 ffff907d9170d9f8 17106 lru 00000080:00200000:0.0:1787826337.144639:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000094e2aae9 17106: 1< 0 - - > 2< 70066176 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144642:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000dfc88ae4 00000080:00200000:0.0:1787826337.144643:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 211. 00000080:00200000:0.0:1787826337.144644:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17107 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144659:0:68279:0:(rw26.c:875:ll_write_end()) pos 70070272, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144663:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000fdda6b35[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17107] 00000080:00200000:0.0:1787826337.144666:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000e28bf9cb 17ffffc0001001 3:0 ffff907d9170d570 17107 lru 00000080:00200000:0.0:1787826337.144670:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000a55b8e44 17107: 1< 0 - - > 2< 70070272 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144672:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000fdda6b35 00000080:00200000:0.0:1787826337.144673:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 212. 00000080:00200000:0.0:1787826337.144675:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17108 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144690:0:68279:0:(rw26.c:875:ll_write_end()) pos 70074368, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144694:0:68279:0:(rw26.c:933:ll_write_end()) page@000000004d0b4a46[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17108] 00000080:00200000:0.0:1787826337.144696:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000037df7620 17ffffc0001001 3:0 ffff907d9170d828 17108 lru 00000080:00200000:0.0:1787826337.144702:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000df826aff 17108: 1< 0 - - > 2< 70074368 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144704:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000004d0b4a46 00000080:00200000:0.0:1787826337.144706:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 213. 00000080:00200000:0.0:1787826337.144708:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17109 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144721:0:68279:0:(rw26.c:875:ll_write_end()) pos 70078464, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144725:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000011c9db12[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17109] 00000080:00200000:0.0:1787826337.144728:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000c84adc87 17ffffc0001001 3:0 ffff907d9170dd98 17109 lru 00000080:00200000:0.0:1787826337.144733:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000003718b531 17109: 1< 0 - - > 2< 70078464 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144735:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000011c9db12 00000080:00200000:0.0:1787826337.144737:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 214. 00000080:00200000:0.0:1787826337.144739:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17110 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144753:0:68279:0:(rw26.c:875:ll_write_end()) pos 70082560, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144757:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000058c8c1aa[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17110] 00000080:00200000:0.0:1787826337.144760:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000006222dc64 17ffffc0001001 3:0 ffff907d9170d658 17110 lru 00000080:00200000:0.0:1787826337.144764:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000b59b79bc 17110: 1< 0 - - > 2< 70082560 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144767:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000058c8c1aa 00000080:00200000:0.0:1787826337.144768:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 215. 00000080:00200000:0.0:1787826337.144770:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17111 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144785:0:68279:0:(rw26.c:875:ll_write_end()) pos 70086656, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144789:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000d667e21d[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17111] 00000080:00200000:0.0:1787826337.144791:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000009ef16016 17ffffc0001001 3:0 ffff907d9170d0e8 17111 lru 00000080:00200000:0.0:1787826337.144796:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000009a97d156 17111: 1< 0 - - > 2< 70086656 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144798:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000d667e21d 00000080:00200000:0.0:1787826337.144799:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 216. 00000080:00200000:0.0:1787826337.144801:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17112 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144813:0:68279:0:(rw26.c:875:ll_write_end()) pos 70090752, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144816:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000033ec1382[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17112] 00000080:00200000:0.0:1787826337.144819:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000034de7a53 17ffffc0001001 3:0 ffff907d9170d910 17112 lru 00000080:00200000:0.0:1787826337.144823:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000040ce0a3b 17112: 1< 0 - - > 2< 70090752 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144825:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000033ec1382 00000080:00200000:0.0:1787826337.144826:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 217. 00000080:00200000:0.0:1787826337.144827:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17113 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144838:0:68279:0:(rw26.c:875:ll_write_end()) pos 70094848, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144842:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000b8c2a4fc[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17113] 00000080:00200000:0.0:1787826337.144844:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000bdfb23cb 17ffffc0001001 3:0 ffff907d9170d1d0 17113 lru 00000080:00200000:0.0:1787826337.144848:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000bb81ff9b 17113: 1< 0 - - > 2< 70094848 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144850:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000b8c2a4fc 00000080:00200000:0.0:1787826337.144851:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 218. 00000080:00200000:0.0:1787826337.144853:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17114 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144865:0:68279:0:(rw26.c:875:ll_write_end()) pos 70098944, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144869:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000fc721024[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17114] 00000080:00200000:0.0:1787826337.144871:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000000af8e032 17ffffc0001001 3:0 ffff907d9170d2b8 17114 lru 00000080:00200000:0.0:1787826337.144877:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000026f486b4 17114: 1< 0 - - > 2< 70098944 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144878:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000fc721024 00000080:00200000:0.0:1787826337.144879:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 219. 00000080:00200000:0.0:1787826337.144881:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17115 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144898:0:68279:0:(rw26.c:875:ll_write_end()) pos 70103040, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144902:0:68279:0:(rw26.c:933:ll_write_end()) page@000000008436a9e1[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17115] 00000080:00200000:0.0:1787826337.144905:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000c8ca2701 17ffffc0001021 2:0 ffff907d9170d000 17115 lru 00000080:00200000:0.0:1787826337.144911:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000f26e3d00 17115: 1< 0 - - > 2< 70103040 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144913:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000008436a9e1 00000080:00200000:0.0:1787826337.144913:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 220. 00000080:00200000:0.0:1787826337.144915:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17116 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144929:0:68279:0:(rw26.c:875:ll_write_end()) pos 70107136, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144933:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000a61b7abf[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17116] 00000080:00200000:0.0:1787826337.144936:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000004e763602 17ffffc0001001 3:0 ffff907d9170dae0 17116 lru 00000080:00200000:0.0:1787826337.144941:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000006ff2c99d 17116: 1< 0 - - > 2< 70107136 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144943:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000a61b7abf 00000080:00200000:0.0:1787826337.144944:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 221. 00000080:00200000:0.0:1787826337.144946:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17117 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144959:0:68279:0:(rw26.c:875:ll_write_end()) pos 70111232, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144964:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000ea79f746[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17117] 00000080:00200000:0.0:1787826337.144966:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000094298364 17ffffc0001001 3:0 ffff907d87c450e8 17117 lru 00000080:00200000:0.0:1787826337.144970:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000e8ae46eb 17117: 1< 0 - - > 2< 70111232 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.144973:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000ea79f746 00000080:00200000:0.0:1787826337.144973:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 222. 00000080:00200000:0.0:1787826337.144976:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17118 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.144989:0:68279:0:(rw26.c:875:ll_write_end()) pos 70115328, len 4096, copied 4096 00000080:00200000:0.0:1787826337.144993:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000964d925f[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17118] 00000080:00200000:0.0:1787826337.144996:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000a73a9a5c 17ffffc0001001 3:0 ffff907d87c45910 17118 lru 00000080:00200000:0.0:1787826337.145001:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000b5f9c392 17118: 1< 0 - - > 2< 70115328 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.145003:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000964d925f 00000080:00200000:0.0:1787826337.145004:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 223. 00000080:00200000:0.0:1787826337.145006:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17119 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.145019:0:68279:0:(rw26.c:875:ll_write_end()) pos 70119424, len 4096, copied 4096 00000080:00200000:0.0:1787826337.145023:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000017f5e081[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17119] 00000080:00200000:0.0:1787826337.145036:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000e0ac4978 17ffffc0001001 3:0 ffff907d87c451d0 17119 lru 00000080:00200000:0.0:1787826337.145042:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000de004ecb 17119: 1< 0 - - > 2< 70119424 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.145044:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000017f5e081 00000080:00200000:0.0:1787826337.145045:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 224. 00000080:00200000:0.0:1787826337.145048:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17120 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.145061:0:68279:0:(rw26.c:875:ll_write_end()) pos 70123520, len 4096, copied 4096 00000080:00200000:0.0:1787826337.145066:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000da99d85b[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17120] 00000080:00200000:0.0:1787826337.145069:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000f9c42f8d 17ffffc0001001 3:0 ffff907d87c452b8 17120 lru 00000080:00200000:0.0:1787826337.145074:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000009caace00 17120: 1< 0 - - > 2< 70123520 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.145076:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000da99d85b 00000080:00200000:0.0:1787826337.145077:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 225. 00000080:00200000:0.0:1787826337.145079:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17121 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.145092:0:68279:0:(rw26.c:875:ll_write_end()) pos 70127616, len 4096, copied 4096 00000080:00200000:0.0:1787826337.145097:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000035704dcc[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17121] 00000080:00200000:0.0:1787826337.145100:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000118560c2 17ffffc0001001 3:0 ffff907d87c45000 17121 lru 00000080:00200000:0.0:1787826337.145105:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000a1294782 17121: 1< 0 - - > 2< 70127616 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.145107:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000035704dcc 00000080:00200000:0.0:1787826337.145108:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 226. 00000080:00200000:0.0:1787826337.145110:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17122 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.145125:0:68279:0:(rw26.c:875:ll_write_end()) pos 70131712, len 4096, copied 4096 00000080:00200000:0.0:1787826337.145130:0:68279:0:(rw26.c:933:ll_write_end()) page@000000006d7be841[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17122] 00000080:00200000:0.0:1787826337.145133:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000021362889 17ffffc0001001 3:0 ffff907d87c45ae0 17122 lru 00000080:00200000:0.0:1787826337.145138:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000728c00d4 17122: 1< 0 - - > 2< 70131712 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.145140:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000006d7be841 00000080:00200000:0.0:1787826337.145141:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 227. 00000080:00200000:0.0:1787826337.145143:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17123 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.145156:0:68279:0:(rw26.c:875:ll_write_end()) pos 70135808, len 4096, copied 4096 00000080:00200000:0.0:1787826337.145160:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000027aaa557[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17123] 00000080:00200000:0.0:1787826337.145163:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000001c01a4ca 17ffffc0001001 3:0 ffff907d87c45e80 17123 lru 00000080:00200000:0.0:1787826337.145168:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000516267e3 17123: 1< 0 - - > 2< 70135808 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.145170:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000027aaa557 00000080:00200000:0.0:1787826337.145171:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 228. 00000080:00200000:0.0:1787826337.145173:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17124 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.148541:0:68279:0:(rw26.c:875:ll_write_end()) pos 70139904, len 4096, copied 4096 00000080:00200000:0.0:1787826337.148558:0:68279:0:(rw26.c:933:ll_write_end()) page@000000003839473d[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17124] 00000080:00200000:0.0:1787826337.148561:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000017b5143d 17ffffc0001001 3:0 ffff907d87c453a0 17124 lru 00000080:00200000:0.0:1787826337.148569:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000fc6a307a 17124: 1< 0 - - > 2< 70139904 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.148572:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000003839473d 00000080:00200000:0.0:1787826337.148573:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 229. 00000080:00200000:0.0:1787826337.148579:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17125 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.150568:0:68279:0:(rw26.c:875:ll_write_end()) pos 70144000, len 4096, copied 4096 00000080:00200000:0.0:1787826337.150591:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000bd6f4077[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17125] 00000080:00200000:0.0:1787826337.150595:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000003661a55d 17ffffc0001001 3:0 ffff907d87c45488 17125 lru 00000080:00200000:0.0:1787826337.150628:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000000c82bffa 17125: 1< 0 - - > 2< 70144000 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.150632:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000bd6f4077 00000080:00200000:0.0:1787826337.150634:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 230. 00000080:00200000:0.0:1787826337.150639:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17126 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.150691:0:68279:0:(rw26.c:875:ll_write_end()) pos 70148096, len 4096, copied 4096 00000080:00200000:0.0:1787826337.150697:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000092e2152e[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17126] 00000080:00200000:0.0:1787826337.150700:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000000c6a3d13 17ffffc0001001 3:0 ffff907d87c45740 17126 lru 00000080:00200000:0.0:1787826337.150706:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000a81d0813 17126: 1< 0 - - > 2< 70148096 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.150709:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000092e2152e 00000080:00200000:0.0:1787826337.150710:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 231. 00000080:00200000:0.0:1787826337.150712:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17127 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.150732:0:68279:0:(rw26.c:875:ll_write_end()) pos 70152192, len 4096, copied 4096 00000080:00200000:0.0:1787826337.150737:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000e68a1975[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17127] 00000080:00200000:0.0:1787826337.150739:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000009527cede 17ffffc0001001 3:0 ffff907d87c45bc8 17127 lru 00000080:00200000:0.0:1787826337.150744:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000006cbd237d 17127: 1< 0 - - > 2< 70152192 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.150746:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000e68a1975 00000080:00200000:0.0:1787826337.150747:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 232. 00000080:00200000:0.0:1787826337.150748:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17128 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.150768:0:68279:0:(rw26.c:875:ll_write_end()) pos 70156288, len 4096, copied 4096 00000080:00200000:0.0:1787826337.150772:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000051d017ca[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17128] 00000080:00200000:0.0:1787826337.150775:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000023dcaaed 17ffffc0001001 3:0 ffff907d87c45cb0 17128 lru 00000080:00200000:0.0:1787826337.150780:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000002bfb999f 17128: 1< 0 - - > 2< 70156288 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.150782:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000051d017ca 00000080:00200000:0.0:1787826337.150783:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 233. 00000080:00200000:0.0:1787826337.150785:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17129 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.150806:0:68279:0:(rw26.c:875:ll_write_end()) pos 70160384, len 4096, copied 4096 00000080:00200000:0.0:1787826337.150810:0:68279:0:(rw26.c:933:ll_write_end()) page@000000008398bc96[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17129] 00000080:00200000:0.0:1787826337.150813:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000006dda13c8 17ffffc0001001 3:0 ffff907d87c459f8 17129 lru 00000080:00200000:0.0:1787826337.150817:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000dcb0f211 17129: 1< 0 - - > 2< 70160384 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.150825:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000008398bc96 00000080:00200000:0.0:1787826337.150826:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 234. 00000080:00200000:0.0:1787826337.150828:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17130 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.150848:0:68279:0:(rw26.c:875:ll_write_end()) pos 70164480, len 4096, copied 4096 00000080:00200000:0.0:1787826337.150858:0:68279:0:(rw26.c:933:ll_write_end()) page@000000005d688c9a[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17130] 00000080:00200000:0.0:1787826337.150861:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000ffc6aeab 17ffffc0001021 2:0 ffff907d87c45570 17130 lru 00000080:00200000:0.0:1787826337.150866:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000053e1abb6 17130: 1< 0 - - > 2< 70164480 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.150867:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000005d688c9a 00000080:00200000:0.0:1787826337.150868:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 235. 00000080:00200000:0.0:1787826337.150870:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17131 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.150884:0:68279:0:(rw26.c:875:ll_write_end()) pos 70168576, len 4096, copied 4096 00000080:00200000:0.0:1787826337.150894:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000cba07d95[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17131] 00000080:00200000:0.0:1787826337.150897:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000068f0ba8b 17ffffc0001001 3:0 ffff907d87c45828 17131 lru 00000080:00200000:0.0:1787826337.150901:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000b38e83e9 17131: 1< 0 - - > 2< 70168576 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.150903:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000cba07d95 00000080:00200000:0.0:1787826337.150904:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 236. 00000080:00200000:0.0:1787826337.150906:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17132 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.150924:0:68279:0:(rw26.c:875:ll_write_end()) pos 70172672, len 4096, copied 4096 00000080:00200000:0.0:1787826337.150929:0:68279:0:(rw26.c:933:ll_write_end()) page@000000000d7665ba[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17132] 00000080:00200000:0.0:1787826337.150932:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000ef4240d8 17ffffc0001001 3:0 ffff907d87c45d98 17132 lru 00000080:00200000:0.0:1787826337.150937:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000018bf3bc5 17132: 1< 0 - - > 2< 70172672 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.150939:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000000d7665ba 00000080:00200000:0.0:1787826337.150940:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 237. 00000080:00200000:0.0:1787826337.150942:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17133 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.154758:0:68279:0:(rw26.c:875:ll_write_end()) pos 70176768, len 4096, copied 4096 00000080:00200000:0.0:1787826337.154775:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000a273f266[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17133] 00000080:00200000:0.0:1787826337.154780:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000007ae8d7e 17ffffc0001001 3:0 ffff907d87c45658 17133 lru 00000080:00200000:0.0:1787826337.154788:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000a03e7e07 17133: 1< 0 - - > 2< 70176768 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.154791:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000a273f266 00000080:00200000:0.0:1787826337.154792:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 238. 00000080:00200000:0.0:1787826337.154798:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17134 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.154845:0:68279:0:(rw26.c:875:ll_write_end()) pos 70180864, len 4096, copied 4096 00000080:00200000:0.0:1787826337.154850:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000989ad40e[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17134] 00000080:00200000:0.0:1787826337.154853:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000000305e982 17ffffc0001001 3:0 ffff907d85559910 17134 lru 00000080:00200000:0.0:1787826337.154858:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000ecd7894c 17134: 1< 0 - - > 2< 70180864 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.154860:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000989ad40e 00000080:00200000:0.0:1787826337.154861:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 239. 00000080:00200000:0.0:1787826337.154863:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17135 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.154879:0:68279:0:(rw26.c:875:ll_write_end()) pos 70184960, len 4096, copied 4096 00000080:00200000:0.0:1787826337.154883:0:68279:0:(rw26.c:933:ll_write_end()) page@000000004a0366bd[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17135] 00000080:00200000:0.0:1787826337.154885:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000087bdfbdc 17ffffc0001001 3:0 ffff907d855591d0 17135 lru 00000080:00200000:0.0:1787826337.154890:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000daf0f7fe 17135: 1< 0 - - > 2< 70184960 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.154892:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000004a0366bd 00000080:00200000:0.0:1787826337.154893:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 240. 00000080:00200000:0.0:1787826337.154895:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17136 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.154910:0:68279:0:(rw26.c:875:ll_write_end()) pos 70189056, len 4096, copied 4096 00000080:00200000:0.0:1787826337.154914:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000a187064f[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17136] 00000080:00200000:0.0:1787826337.154916:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000a4379756 17ffffc0001001 3:0 ffff907d855592b8 17136 lru 00000080:00200000:0.0:1787826337.154922:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000a2f855cb 17136: 1< 0 - - > 2< 70189056 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.154925:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000a187064f 00000080:00200000:0.0:1787826337.154926:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 241. 00000080:00200000:0.0:1787826337.154928:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17137 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.154941:0:68279:0:(rw26.c:875:ll_write_end()) pos 70193152, len 4096, copied 4096 00000080:00200000:0.0:1787826337.154945:0:68279:0:(rw26.c:933:ll_write_end()) page@000000001095740e[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17137] 00000080:00200000:0.0:1787826337.154948:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000059b7f51d 17ffffc0001001 3:0 ffff907d85559000 17137 lru 00000080:00200000:0.0:1787826337.154953:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000004268df57 17137: 1< 0 - - > 2< 70193152 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.154955:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000001095740e 00000080:00200000:0.0:1787826337.154957:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 242. 00000080:00200000:0.0:1787826337.154958:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17138 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.154972:0:68279:0:(rw26.c:875:ll_write_end()) pos 70197248, len 4096, copied 4096 00000080:00200000:0.0:1787826337.154977:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000ee3d5c8f[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17138] 00000080:00200000:0.0:1787826337.154979:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000be3128fb 17ffffc0001001 3:0 ffff907d85559ae0 17138 lru 00000080:00200000:0.0:1787826337.154984:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000618cd9b0 17138: 1< 0 - - > 2< 70197248 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.154986:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000ee3d5c8f 00000080:00200000:0.0:1787826337.154987:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 243. 00000080:00200000:0.0:1787826337.154989:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17139 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.155001:0:68279:0:(rw26.c:875:ll_write_end()) pos 70201344, len 4096, copied 4096 00000080:00200000:0.0:1787826337.155005:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000196146f1[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17139] 00000080:00200000:0.0:1787826337.155008:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000002ede57e0 17ffffc0001001 3:0 ffff907d85559e80 17139 lru 00000080:00200000:0.0:1787826337.155014:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000009eaeca28 17139: 1< 0 - - > 2< 70201344 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.155016:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000196146f1 00000080:00200000:0.0:1787826337.155017:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 244. 00000080:00200000:0.0:1787826337.155019:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17140 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.155043:0:68279:0:(rw26.c:875:ll_write_end()) pos 70205440, len 4096, copied 4096 00000080:00200000:0.0:1787826337.155047:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000055d75bdc[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17140] 00000080:00200000:0.0:1787826337.155050:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000000a7a0da1 17ffffc0001001 3:0 ffff907d855593a0 17140 lru 00000080:00200000:0.0:1787826337.155056:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000d01127d1 17140: 1< 0 - - > 2< 70205440 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.155058:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000055d75bdc 00000080:00200000:0.0:1787826337.155059:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 245. 00000080:00200000:0.0:1787826337.155061:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17141 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.155075:0:68279:0:(rw26.c:875:ll_write_end()) pos 70209536, len 4096, copied 4096 00000080:00200000:0.0:1787826337.155080:0:68279:0:(rw26.c:933:ll_write_end()) page@000000003e99101e[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17141] 00000080:00200000:0.0:1787826337.155082:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000095c159be 17ffffc0001001 3:0 ffff907d85559488 17141 lru 00000080:00200000:0.0:1787826337.155088:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000007d36d0f2 17141: 1< 0 - - > 2< 70209536 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.155090:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000003e99101e 00000080:00200000:0.0:1787826337.155091:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 246. 00000080:00200000:0.0:1787826337.155093:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17142 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.155107:0:68279:0:(rw26.c:875:ll_write_end()) pos 70213632, len 4096, copied 4096 00000080:00200000:0.0:1787826337.155111:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000c3203147[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17142] 00000080:00200000:0.0:1787826337.155114:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000016d394e2 17ffffc0001001 3:0 ffff907d85559740 17142 lru 00000080:00200000:0.0:1787826337.155120:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000d60be490 17142: 1< 0 - - > 2< 70213632 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.155122:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000c3203147 00000080:00200000:0.0:1787826337.155123:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 247. 00000080:00200000:0.0:1787826337.155124:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17143 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.155139:0:68279:0:(rw26.c:875:ll_write_end()) pos 70217728, len 4096, copied 4096 00000080:00200000:0.0:1787826337.155144:0:68279:0:(rw26.c:933:ll_write_end()) page@000000007a2be715[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17143] 00000080:00200000:0.0:1787826337.155146:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000555e896f 17ffffc0001001 3:0 ffff907d85559bc8 17143 lru 00000080:00200000:0.0:1787826337.155151:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@0000000098028c86 17143: 1< 0 - - > 2< 70217728 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.155154:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000007a2be715 00000080:00200000:0.0:1787826337.155155:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 248. 00000080:00200000:0.0:1787826337.155157:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17144 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.155172:0:68279:0:(rw26.c:875:ll_write_end()) pos 70221824, len 4096, copied 4096 00000080:00200000:0.0:1787826337.155176:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000c6dd81e7[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17144] 00000080:00200000:0.0:1787826337.155178:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000aa1e6a41 17ffffc0001001 3:0 ffff907d85559cb0 17144 lru 00000080:00200000:0.0:1787826337.155183:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000481580cc 17144: 1< 0 - - > 2< 70221824 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.155185:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000c6dd81e7 00000080:00200000:0.0:1787826337.155186:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 249. 00000080:00200000:0.0:1787826337.155187:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17145 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.155206:0:68279:0:(rw26.c:875:ll_write_end()) pos 70225920, len 4096, copied 4096 00000080:00200000:0.0:1787826337.155210:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000439db201[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17145] 00000080:00200000:0.0:1787826337.155213:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000adc24234 17ffffc0001021 2:0 ffff907d855599f8 17145 lru 00000080:00200000:0.0:1787826337.155219:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@00000000d1983649 17145: 1< 0 - - > 2< 70225920 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.155222:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000439db201 00000080:00200000:0.0:1787826337.155223:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 250. 00000080:00200000:0.0:1787826337.155225:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17146 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.155239:0:68279:0:(rw26.c:875:ll_write_end()) pos 70230016, len 4096, copied 4096 00000080:00200000:0.0:1787826337.155244:0:68279:0:(rw26.c:933:ll_write_end()) page@000000000cfd23aa[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17146] 00000080:00200000:0.0:1787826337.155246:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000082550a0f 17ffffc0001001 3:0 ffff907d85559570 17146 lru 00000080:00200000:0.0:1787826337.155251:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000006c153483 17146: 1< 0 - - > 2< 70230016 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.155254:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000000cfd23aa 00000080:00200000:0.0:1787826337.155254:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 251. 00000080:00200000:0.0:1787826337.155256:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17147 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.155269:0:68279:0:(rw26.c:875:ll_write_end()) pos 70234112, len 4096, copied 4096 00000080:00200000:0.0:1787826337.155274:0:68279:0:(rw26.c:933:ll_write_end()) page@000000006ead8db7[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17147] 00000080:00200000:0.0:1787826337.155276:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000459ad14b 17ffffc0001001 3:0 ffff907d85559828 17147 lru 00000080:00200000:0.0:1787826337.155282:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000008440b05d 17147: 1< 0 - - > 2< 70234112 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.155284:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000006ead8db7 00000080:00200000:0.0:1787826337.155285:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 252. 00000080:00200000:0.0:1787826337.155474:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17148 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.155492:0:68279:0:(rw26.c:875:ll_write_end()) pos 70238208, len 4096, copied 4096 00000080:00200000:0.0:1787826337.155497:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000025978d4a[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17148] 00000080:00200000:0.0:1787826337.155499:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @000000005cbf44dd 17ffffc0001001 3:0 ffff907d85559d98 17148 lru 00000080:00200000:0.0:1787826337.155504:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000006ac96a23 17148: 1< 0 - - > 2< 70238208 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.155506:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000025978d4a 00000080:00200000:0.0:1787826337.155507:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 253. 00000080:00200000:0.0:1787826337.155509:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17149 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.155521:0:68279:0:(rw26.c:875:ll_write_end()) pos 70242304, len 4096, copied 4096 00000080:00200000:0.0:1787826337.155525:0:68279:0:(rw26.c:933:ll_write_end()) page@0000000054185b9c[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17149] 00000080:00200000:0.0:1787826337.155527:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @0000000085781a5f 17ffffc0001001 3:0 ffff907d85559658 17149 lru 00000080:00200000:0.0:1787826337.155531:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000000a2211a6 17149: 1< 0 - - > 2< 70242304 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.155533:0:68279:0:(rw26.c:933:ll_write_end()) end page@0000000054185b9c 00000080:00200000:0.0:1787826337.155534:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 254. 00000080:00200000:0.0:1787826337.155535:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17150 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.155548:0:68279:0:(rw26.c:875:ll_write_end()) pos 70246400, len 4096, copied 4096 00000080:00200000:0.0:1787826337.155553:0:68279:0:(rw26.c:933:ll_write_end()) page@000000000df05764[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17150] 00000080:00200000:0.0:1787826337.155555:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000ca9b1dda 17ffffc0001001 3:0 ffff907d855590e8 17150 lru 00000080:00200000:0.0:1787826337.155559:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000002aae7434 17150: 1< 0 - - > 2< 70246400 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.155561:0:68279:0:(rw26.c:933:ll_write_end()) end page@000000000df05764 00000080:00200000:0.0:1787826337.155562:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 255. 00000080:00200000:0.0:1787826337.155564:0:68279:0:(rw26.c:681:ll_write_begin()) Writing 17151 of 0 to 4096 bytes 00000080:00200000:0.0:1787826337.155578:0:68279:0:(rw26.c:875:ll_write_end()) pos 70250496, len 4096, copied 4096 00000080:00200000:0.0:1787826337.155582:0:68279:0:(rw26.c:933:ll_write_end()) page@00000000e9df48cc[3 00000000bd7162fe 2 1 000000008ccf9238 65536 0 17151] 00000080:00200000:0.0:1787826337.155585:0:68279:0:(rw26.c:933:ll_write_end()) vmpage @00000000146029b7 17ffffc0001001 3:0 ffff907d909f9000 17151 lru 00000080:00200000:0.0:1787826337.155589:0:68279:0:(rw26.c:933:ll_write_end()) osc-page@000000007135dab1 17151: 1< 0 - - > 2< 70250496 0 0 0x0 0x0 | 000000005f5e6cd2 00000000ef0ebe13 > 3< 0 0 > 4< 0 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1787826337.155591:0:68279:0:(rw26.c:933:ll_write_end()) end page@00000000e9df48cc 00000080:00200000:0.0:1787826337.155592:0:68279:0:(rw26.c:933:ll_write_end()) queued cl_page: 256. 00000080:00200000:0.0:1787826337.155597:0:68279:0:(vvp_io.c:1214:vvp_io_write_commit()) commit async pages: 256, from 0, to 4096 prio 0 00000008:00010000:0.0:1787826337.155693:0:68279:0:(osc_cache.c:858:osc_extent_find()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.155718:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.155729:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.155738:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.155746:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.155755:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.155769:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.155777:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.155786:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.155794:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.155803:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156454:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156466:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156474:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156482:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826337.156499:0:68279:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93315d38, count 15, dirtied 15 00000008:00010000:0.0:1787826337.156506:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156514:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156522:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156529:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156538:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156546:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156553:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156562:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156575:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156582:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156590:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156598:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156732:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156743:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156753:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826337.156767:0:68279:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93315d38, count 15, dirtied 15 00000008:00010000:0.0:1787826337.156773:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156788:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156797:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156805:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156858:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156866:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156875:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156884:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156892:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156901:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156910:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156919:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156927:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156935:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156942:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826337.156957:0:68279:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93315d38, count 15, dirtied 15 00000008:00010000:0.0:1787826337.156963:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156972:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156980:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.156992:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157002:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157011:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157020:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157039:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157048:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157057:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157065:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157072:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157081:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157091:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157101:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826337.157113:0:68279:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93315d38, count 15, dirtied 15 00000008:00010000:0.0:1787826337.157117:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157126:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157134:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157142:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157149:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157163:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157172:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157181:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157189:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157198:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157206:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157213:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157221:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157228:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157236:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826337.157249:0:68279:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93315d38, count 15, dirtied 15 00000008:00010000:0.0:1787826337.157254:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157263:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157272:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157281:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157764:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157775:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157784:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157798:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157805:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157887:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157896:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157904:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157913:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157921:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157930:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826337.157944:0:68279:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93315d38, count 15, dirtied 15 00000008:00010000:0.0:1787826337.157950:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157958:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157966:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157973:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157981:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157989:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.157996:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158005:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158012:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158037:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158047:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158055:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158063:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158071:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158080:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826337.158095:0:68279:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93315d38, count 15, dirtied 15 00000008:00010000:0.0:1787826337.158101:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158109:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158123:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158132:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158139:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158148:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158156:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158165:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158174:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158251:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158260:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158270:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158278:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158286:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158553:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826337.158570:0:68279:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93315d38, count 15, dirtied 15 00000008:00010000:0.0:1787826337.158577:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158585:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158592:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.158599:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.159218:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.159228:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.159235:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.159243:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.159251:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.159258:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.159266:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.159273:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.159280:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.159300:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.159311:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826337.159330:0:68279:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93315d38, count 15, dirtied 15 00000008:00010000:0.0:1787826337.159336:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.159345:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.159352:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.159360:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.159369:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.159376:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.159389:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.159409:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160537:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160555:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160564:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160572:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160588:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160600:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160622:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826337.160639:0:68279:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93315d38, count 15, dirtied 15 00000008:00010000:0.0:1787826337.160647:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160656:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160665:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160674:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160749:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160766:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160775:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160783:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160805:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160932:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160946:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160954:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160962:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160969:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.160984:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826337.160997:0:68279:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93315d38, count 15, dirtied 15 00000008:00010000:0.0:1787826337.161003:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161023:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161047:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161058:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161069:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161079:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161087:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161103:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161111:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161119:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161133:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161141:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161148:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161162:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161172:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826337.161235:0:68279:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93315d38, count 15, dirtied 15 00000008:00010000:0.0:1787826337.161241:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161250:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161258:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161271:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161279:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161397:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161410:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161527:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161536:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161550:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161558:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161570:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161580:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161634:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161645:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826337.161660:0:68279:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93315d38, count 15, dirtied 15 00000008:00010000:0.0:1787826337.161666:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161741:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161753:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161761:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161769:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161783:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161791:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161800:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161883:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161894:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161903:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161912:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161919:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161927:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161943:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826337.161957:0:68279:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93315d38, count 15, dirtied 15 00000008:00010000:0.0:1787826337.161962:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161977:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161985:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.161999:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162010:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162018:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162042:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162055:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162070:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162079:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162088:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162096:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162111:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162119:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162128:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826337.162148:0:68279:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93315d38, count 15, dirtied 15 00000008:00010000:0.0:1787826337.162154:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162162:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162169:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162933:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162942:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162951:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162960:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162969:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162977:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162988:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.162996:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.163003:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.163011:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.163019:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.163036:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826337.163052:0:68279:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93315d38, count 15, dirtied 15 00000008:00010000:0.0:1787826337.163058:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.163065:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.163078:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.163085:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.163093:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.163100:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.163108:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.163115:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.163123:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.163131:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.163137:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.163144:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.163151:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.163158:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826337.163166:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826337.163188:0:68279:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93315d38, count 15, dirtied 15 00000008:00010000:0.0:1787826337.163194:0:68279:0:(osc_cache.c:2567:osc_queue_async_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:0.0:1787826337.163203:0:68279:0:(vvp_io.c:1127:vvp_set_batch_dirty()) mapping ffff907d93315d38, count 1, dirtied 1 00000080:00200000:0.0:1787826337.163206:0:68279:0:(vvp_io.c:1237:vvp_io_write_commit()) Committed 256 pages 1048576 bytes, tot: 1048576 00000080:00200000:0.0:1787826337.163211:0:68279:0:(vvp_io.c:1421:vvp_io_write_start()) f204d.sanity-flr: write bytes 0, result: 1048576 00000020:00010000:0.0:1787826337.163227:0:68279:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_release():245 00000020:00010000:0.0:1787826337.163231:0:68279:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1787826337.163234:0:68279:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da032e9b0 (ffff907d884a71b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1787826337.163241:0:68279:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826337.163256:0:68279:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:0.0:1787826337.163272:0:68279:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1787826337.163284:0:68279:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da032e9b0 (ffff907d884a7d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1787826337.163554:0:68279:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000080:00200000:0.0:1787826337.163562:0:68279:0:(file.c:2207:ll_file_io_generic()) Range unlock [16896, 17151] 00000080:00200000:0.0:1787826337.163571:0:68279:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 6 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1787826337.163580:0:68279:0:(file.c:2240:ll_file_io_generic()) f204d.sanity-flr: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:0.0:1787826337.163594:0:68279:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 0 00000080:00200000:0.0:1787826337.163599:0:68279:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:0.0:1787826337.163768:0:68279:0:(file.c:479:ll_file_release()) START file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88), flags 100002 00000002:00100000:0.0:1787826337.163834:0:68279:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d90846580 x1874670011443456/t8589943001(8589943001) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 984/1056 e 0 to 0 dl 1787826355 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1787826337.163847:0:68279:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1787826337.163852:0:68279:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90847840 x1874670011444224/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1787826337.163874:0:68279:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:68279:1874670011444224:192.168.203.104@tcp:35:dd.0 00000100:00100000:0.0:1787826337.164049:0:68279:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd300 going to sleep for 11 seconds 00000100:00100000:0.0:1787826337.170113:0:68279:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:68279:1874670011444224:192.168.203.104@tcp:35:dd.0 00000002:00100000:0.0:1787826337.170144:0:68279:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d90846580 x1874670011443456/t8589943001(8589943001) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 984/1056 e 0 to 0 dl 1787826355 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00010000:00010000:3.0:1787826337.653574:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826338.549925:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011444352:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826338.550073:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011444608:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826338.550114:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1787826338.550121:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011444480:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826338.550146:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011444736:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826338.553694:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011444352:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826338.560077:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011444480:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826338.560099:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011444736:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826338.560110:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011444608:192.168.203.104@tcp:400:kworker.0 00010000:00010000:0.0:1787826338.677199:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826339.704779:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826340.725152:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826341.751162:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826342.772809:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826343.669576:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011444992:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826343.669768:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011445248:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826343.669815:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1787826343.669837:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2348:1874670011444864:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826343.669869:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011445120:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826343.672294:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011445248:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826343.675371:0:2348:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589943003 gen 1 00000100:00100000:1.0:1787826343.675389:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011444992:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826343.675432:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011445120:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826343.678198:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2348:1874670011444864:192.168.203.104@tcp:400:kworker.0 00010000:00010000:0.0:1787826343.798182:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826344.820791:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826345.850340:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826346.874160:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826347.893257:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826348.789316:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011445376:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826348.789416:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011445632:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826348.789449:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1787826348.789456:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011445504:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826348.789481:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011445760:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826348.792235:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011445504:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826348.802271:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011445632:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826348.802319:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011445760:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826348.802331:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011445376:192.168.203.104@tcp:400:kworker.0 00010000:00010000:1.0:1787826348.916893:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1787826349.541586:0:68302:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826349.541631:0:68302:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826349.541649:0:68302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:3.0:1787826349.541664:0:68302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:3.0:1787826349.541672:0:68302:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826349.541680:0:68302:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826349.541683:0:68302:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:3.0:1787826349.541690:0:68302:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142b87bf8 00010000:00010000:3.0:1787826349.541694:0:68302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:3.0:1787826349.541703:0:68302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:3.0:1787826349.541718:0:68302:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826349.541725:0:68302:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:3.0:1787826349.541730:0:68302:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:3.0:1787826349.541740:0:68302:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), file ffff907d83cd0480, open_flags 18800, mode 0 opened 0 00000080:00200000:3.0:1787826349.541746:0:68302:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), intent=open 00000002:00010000:3.0:1787826349.541757:0:68302:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x205:0x0], intent: open flags 0304001 00010000:00010000:3.0:1787826349.541803:0:68302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9a0 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.541814:0:68302:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9a0 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826349.541831:0:68302:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826349.541842:0:68302:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d835261c0 x1874670011446016/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826349.541850:0:68302:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9a0 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826349.541868:0:68302:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68302:1874670011446016:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826349.545592:0:68302:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d866e9240 going to sleep for 13 seconds 00000100:00100000:3.0:1787826349.556266:0:68302:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68302:1874670011446016:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826349.556290:0:68302:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9a0 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.556304:0:68302:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9a0 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.556311:0:68302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9a0 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.556317:0:68302:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9a0 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.556325:0:68302:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9a0 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.556341:0:68302:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8629c200 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826349.556347:0:68302:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9a0 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.556353:0:68302:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9a0 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.556361:0:68302:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9a0 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.556368:0:68302:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826349.556374:0:68302:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9a0 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.556381:0:68302:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9a0 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826349.556399:0:68302:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d835261c0 x1874670011446016/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826367 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826349.556413:0:68302:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x205:0x0] suppgids 0 -1: rc -20 00000080:00200000:3.0:1787826349.556534:0:68302:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826349.556552:0:68302:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826349.556564:0:68302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:3.0:1787826349.556575:0:68302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:3.0:1787826349.556583:0:68302:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826349.556588:0:68302:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826349.556591:0:68302:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:3.0:1787826349.556596:0:68302:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142b87bf8 00010000:00010000:3.0:1787826349.556600:0:68302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:3.0:1787826349.556636:0:68302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:3.0:1787826349.556649:0:68302:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826349.556657:0:68302:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=267 00000080:00200000:3.0:1787826349.556663:0:68302:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 44 00000080:00200008:3.0:1787826349.556671:0:68302:0:(file.c:1090:ll_file_open()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00000002:00010000:3.0:1787826349.556686:0:68302:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x205:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:3.0:1787826349.556728:0:68302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9a7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.556748:0:68302:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9a7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826349.556758:0:68302:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826349.556764:0:68302:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d835261c0 x1874670011446272/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826349.556775:0:68302:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9a7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826349.556795:0:68302:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68302:1874670011446272:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826349.556972:0:68302:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d866e9240 going to sleep for 13 seconds 00000100:00100000:3.0:1787826349.563012:0:68302:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68302:1874670011446272:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826349.563046:0:68302:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9a7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.563065:0:68302:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9a7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.563072:0:68302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9a7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.563081:0:68302:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9a7 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.563090:0:68302:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9a7 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.563096:0:68302:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8629dc00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826349.563101:0:68302:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9a7 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.563108:0:68302:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9a7 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.563115:0:68302:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9a7 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.563123:0:68302:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826349.563131:0:68302:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9a7 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.563138:0:68302:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9a7 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68302 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826349.563159:0:68302:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffff907d835261c0 x1874670011446272/t8589943004(8589943004) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826367 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1787826349.563181:0:68302:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d835261c0 x1874670011446272/t8589943004(8589943004) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826367 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826349.563195:0:68302:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1787826349.563406:0:68302:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x205:0x0], updating i_size 4096 00000002:00100000:3.0:1787826349.563430:0:68302:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d835261c0 x1874670011446272/t8589943004(8589943004) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826367 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:3.0:1787826349.563568:0:68302:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908) cmd=c0086915 arg=baa3d0 00000100:00100000:3.0:1787826349.563871:0:68302:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68302:1874670011446400:192.168.203.104@tcp:34:lfs.0 00000100:00100000:3.0:1787826349.563976:0:68302:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d866e9240 going to sleep for 13 seconds 00000100:00100000:3.0:1787826349.568264:0:68302:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68302:1874670011446400:192.168.203.104@tcp:34:lfs.0 00000080:00200008:2.0:1787826349.573076:0:68302:0:(file.c:479:ll_file_release()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00000002:00100000:2.0:1787826349.573185:0:68302:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d835261c0 x1874670011446272/t8589943004(8589943004) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826367 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1787826349.573201:0:68302:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1787826349.573218:0:68302:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d911483c0 x1874670011446656/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1787826349.573249:0:68302:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68302:1874670011446656:192.168.203.104@tcp:35:lfs.0 00000100:00100000:2.0:1787826349.573357:0:68302:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88d28f00 going to sleep for 11 seconds 00000100:00100000:2.0:1787826349.577298:0:68302:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68302:1874670011446656:192.168.203.104@tcp:35:lfs.0 00000002:00100000:2.0:1787826349.577342:0:68302:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d835261c0 x1874670011446272/t8589943004(8589943004) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826367 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1787826349.621203:0:68304:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1787826349.621214:0:68304:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826349.621233:0:68304:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826349.621920:0:68304:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1787826349.621947:0:68304:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:2.0:1787826349.621961:0:68304:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826349.621969:0:68304:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826349.621973:0:68304:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:2.0:1787826349.621981:0:68304:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142b27cb8 00010000:00010000:2.0:1787826349.621984:0:68304:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1787826349.621992:0:68304:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:2.0:1787826349.622101:0:68304:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826349.622112:0:68304:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826349.622120:0:68304:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1787826349.622129:0:68304:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:2.0:1787826349.622135:0:68304:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826349.622139:0:68304:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826349.622142:0:68304:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:2.0:1787826349.622146:0:68304:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142b27bb0 00010000:00010000:2.0:1787826349.622149:0:68304:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1787826349.622156:0:68304:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:2.0:1787826349.622165:0:68304:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:2.0:1787826349.622172:0:68304:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:2.0:1787826349.622187:0:68304:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7fffb75ce75c 00000080:00200008:2.0:1787826349.622198:0:68304:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:3.0:1787826349.626137:0:68307:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826349.626179:0:68307:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826349.626208:0:68307:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:3.0:1787826349.626229:0:68307:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:3.0:1787826349.626237:0:68307:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826349.626251:0:68307:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826349.626255:0:68307:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:3.0:1787826349.626265:0:68307:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142b97bf8 00010000:00010000:3.0:1787826349.626269:0:68307:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:3.0:1787826349.626284:0:68307:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:3.0:1787826349.626302:0:68307:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826349.626310:0:68307:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:3.0:1787826349.626317:0:68307:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:3.0:1787826349.626331:0:68307:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), file ffff907d83cd0480, open_flags 18800, mode 0 opened 0 00000080:00200000:3.0:1787826349.626340:0:68307:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), intent=open 00000002:00010000:3.0:1787826349.626357:0:68307:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x205:0x0], intent: open flags 0304001 00010000:00010000:3.0:1787826349.626426:0:68307:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9ae lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68307 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.626435:0:68307:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9ae lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68307 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826349.626448:0:68307:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826349.626464:0:68307:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d835270c0 x1874670011446912/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826349.626475:0:68307:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9ae lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68307 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826349.626503:0:68307:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68307:1874670011446912:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826349.626967:0:68307:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d866e9240 going to sleep for 13 seconds 00000100:00100000:3.0:1787826349.630865:0:68307:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68307:1874670011446912:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826349.630894:0:68307:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9ae lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68307 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.630912:0:68307:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9ae lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68307 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.630919:0:68307:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9ae lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68307 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.630927:0:68307:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9ae lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68307 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.630937:0:68307:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9ae lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68307 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.630952:0:68307:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8629dc00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826349.630957:0:68307:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9ae lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 68307 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.630965:0:68307:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9ae lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 68307 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.630973:0:68307:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9ae lrc: 5/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 68307 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.630980:0:68307:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826349.630989:0:68307:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9ae lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68307 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.630996:0:68307:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9ae lrc: 0/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68307 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826349.631019:0:68307:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d835270c0 x1874670011446912/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826367 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826349.631080:0:68307:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x205:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826349.639894:0:68307:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826349.639934:0:68307:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826349.639971:0:68307:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:1.0:1787826349.639990:0:68307:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:1.0:1787826349.639999:0:68307:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826349.640008:0:68307:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826349.640012:0:68307:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:1.0:1787826349.640023:0:68307:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142b97bf8 00010000:00010000:1.0:1787826349.640028:0:68307:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:1.0:1787826349.640040:0:68307:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:1.0:1787826349.640059:0:68307:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826349.640069:0:68307:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=267 00000080:00200000:1.0:1787826349.640078:0:68307:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 44 00000080:00200008:1.0:1787826349.640089:0:68307:0:(file.c:1090:ll_file_open()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00000002:00010000:1.0:1787826349.640106:0:68307:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x205:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240304001 00010000:00010000:1.0:1787826349.640173:0:68307:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b9b5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68307 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.640182:0:68307:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b9b5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68307 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826349.640212:0:68307:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826349.640229:0:68307:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a0780 x1874670011447168/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826349.640240:0:68307:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b9b5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68307 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826349.640266:0:68307:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68307:1874670011447168:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826349.640487:0:68307:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d872489c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826349.648573:0:68307:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68307:1874670011447168:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826349.648628:0:68307:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b9b5 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10358 expref: -99 pid: 68307 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.648641:0:68307:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b9b5 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10358 expref: -99 pid: 68307 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.648652:0:68307:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b9b5 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x205:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10358 expref: -99 pid: 68307 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.648664:0:68307:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b9b5 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x205:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10358 expref: -99 pid: 68307 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826349.648680:0:68307:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x200002b, status=0 req@ffff907d908a0780 x1874670011447168/t8589943006(8589943006) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826367 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826349.648693:0:68307:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a0780 x1874670011447168/t8589943006(8589943006) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826367 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826349.648704:0:68307:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b9b5 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x205:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10358 expref: -99 pid: 68307 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.648714:0:68307:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/517 (0 0) 00000080:00200000:1.0:1787826349.648719:0:68307:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826349.648737:0:68307:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x205:0x0], updating i_size 4096 00000080:00010000:1.0:1787826349.648741:0:68307:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x205:0x0](ffff907d93310908) for lock 0xe5f736940357b9b5 00000080:00010000:1.0:1787826349.648749:0:68307:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b9b5 from it ffff9dd142b97bd0 00010000:00010000:1.0:1787826349.648752:0:68307:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b9b5 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x205:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10358 expref: -99 pid: 68307 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.648762:0:68307:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b9b5 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x205:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10358 expref: -99 pid: 68307 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1787826349.648790:0:68307:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a0780 x1874670011447168/t8589943006(8589943006) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826367 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1787826349.648875:0:68307:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908) cmd=c1406916 arg=2271a20 00000100:00100000:1.0:1787826349.648930:0:68307:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68307:1874670011447296:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826349.649019:0:68307:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d872489c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826349.654351:0:68307:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68307:1874670011447296:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826349.654510:0:68307:0:(file.c:479:ll_file_release()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00010000:00010000:1.0:1787826349.654538:0:68307:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b9b5 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x205:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10358 expref: -99 pid: 68307 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787826349.700721:0:68310:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826349.700748:0:68310:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826349.700765:0:68310:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826349.700782:0:68310:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:3.0:1787826349.700791:0:68310:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826349.700989:0:68310:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826349.700994:0:68310:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:3.0:1787826349.701002:0:68310:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142bafbf8 00010000:00010000:3.0:1787826349.701006:0:68310:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:3.0:1787826349.701016:0:68310:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:3.0:1787826349.701033:0:68310:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826349.701042:0:68310:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:3.0:1787826349.701047:0:68310:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:3.0:1787826349.701252:0:68310:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), file ffff907d83529b00, open_flags 18800, mode 0 opened 0 00000080:00200000:3.0:1787826349.701262:0:68310:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), intent=open 00000002:00010000:3.0:1787826349.701277:0:68310:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x205:0x0], intent: open flags 0304001 00010000:00010000:3.0:1787826349.701329:0:68310:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9bc lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 68310 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.701339:0:68310:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9bc lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68310 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826349.701349:0:68310:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826349.701360:0:68310:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d835270c0 x1874670011447552/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826349.701370:0:68310:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9bc lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68310 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826349.701462:0:68310:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68310:1874670011447552:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826349.701874:0:68310:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d866e9240 going to sleep for 13 seconds 00000100:00100000:2.0:1787826349.710837:0:68310:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68310:1874670011447552:192.168.203.104@tcp:101:lfs.0 00010000:00010000:2.0:1787826349.710862:0:68310:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9bc lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68310 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826349.710875:0:68310:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9bc lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 68310 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826349.710883:0:68310:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9bc lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 68310 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826349.710890:0:68310:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9bc lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 68310 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826349.710898:0:68310:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9bc lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 68310 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826349.710904:0:68310:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8629dc00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:2.0:1787826349.710909:0:68310:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9bc lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 68310 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826349.710916:0:68310:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9bc lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 68310 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826349.710924:0:68310:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9bc lrc: 5/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 68310 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826349.710932:0:68310:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1787826349.710949:0:68310:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9bc lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 68310 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826349.710957:0:68310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629dc00/0xe5f736940357b9bc lrc: 0/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 68310 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826349.710976:0:68310:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d835270c0 x1874670011447552/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826367 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1787826349.710990:0:68310:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x205:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1787826349.711114:0:68310:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826349.711132:0:68310:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826349.711158:0:68310:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:2.0:1787826349.711170:0:68310:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:2.0:1787826349.711177:0:68310:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826349.711182:0:68310:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826349.711186:0:68310:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:2.0:1787826349.711191:0:68310:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142bafbf8 00010000:00010000:2.0:1787826349.711194:0:68310:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:2.0:1787826349.711329:0:68310:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:2.0:1787826349.711351:0:68310:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826349.711359:0:68310:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=267 00000080:00200000:2.0:1787826349.711365:0:68310:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 44 00000080:00200008:2.0:1787826349.711373:0:68310:0:(file.c:1090:ll_file_open()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00000080:00200080:2.0:1787826349.711479:0:68310:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908) cmd=c0086915 arg=1dcf3f0 00000100:00100000:2.0:1787826349.711538:0:68310:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68310:1874670011447680:192.168.203.104@tcp:34:lfs.0 00000100:00100000:2.0:1787826349.711658:0:68310:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88d28f00 going to sleep for 13 seconds 00000100:00100000:2.0:1787826349.721034:0:68310:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68310:1874670011447680:192.168.203.104@tcp:34:lfs.0 00000080:00200008:2.0:1787826349.721235:0:68310:0:(file.c:479:ll_file_release()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00010000:00010000:2.0:1787826349.721263:0:68310:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b9b5 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x205:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10358 expref: -99 pid: 68307 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1787826349.765319:0:68312:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826349.765347:0:68312:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826349.765365:0:68312:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00010000:00010000:1.0:1787826349.765382:0:68312:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:1.0:1787826349.765465:0:68312:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826349.765473:0:68312:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826349.765477:0:68312:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:1.0:1787826349.765485:0:68312:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142b9fbf8 00010000:00010000:1.0:1787826349.765489:0:68312:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00010000:00010000:1.0:1787826349.765504:0:68312:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:1.0:1787826349.765589:0:68312:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826349.765598:0:68312:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:1.0:1787826349.765602:0:68312:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:1.0:1787826349.765858:0:68312:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), file ffff907d8500b680, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1787826349.765866:0:68312:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), intent=open 00000002:00010000:1.0:1787826349.765880:0:68312:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x205:0x0], intent: open flags 0304001 00010000:00010000:1.0:1787826349.765933:0:68312:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9c3 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 68312 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.765953:0:68312:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9c3 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68312 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826349.765964:0:68312:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826349.765975:0:68312:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a1e00 x1874670011447936/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826349.765984:0:68312:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9c3 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68312 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826349.766005:0:68312:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68312:1874670011447936:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826349.766228:0:68312:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d872489c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826349.771668:0:68312:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68312:1874670011447936:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826349.771690:0:68312:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9c3 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68312 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.771704:0:68312:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9c3 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 68312 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.771711:0:68312:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9c3 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 68312 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.771717:0:68312:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9c3 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 68312 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.771734:0:68312:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9c3 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 68312 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.771741:0:68312:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0ae00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826349.771745:0:68312:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9c3 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 68312 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.771752:0:68312:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9c3 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 68312 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.771759:0:68312:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9c3 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 68312 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.771766:0:68312:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826349.771773:0:68312:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9c3 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 68312 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.771779:0:68312:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9c3 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 68312 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826349.771797:0:68312:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908a1e00 x1874670011447936/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826367 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826349.771821:0:68312:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x205:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826349.771951:0:68312:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826349.771968:0:68312:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826349.771983:0:68312:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:1.0:1787826349.771995:0:68312:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:1.0:1787826349.772002:0:68312:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826349.772008:0:68312:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826349.772011:0:68312:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:1.0:1787826349.772017:0:68312:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142b9fbf8 00010000:00010000:1.0:1787826349.772020:0:68312:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:1.0:1787826349.772029:0:68312:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:1.0:1787826349.772045:0:68312:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826349.772064:0:68312:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=267 00000080:00200000:1.0:1787826349.772071:0:68312:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 44 00000080:00200008:1.0:1787826349.772079:0:68312:0:(file.c:1090:ll_file_open()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00000080:00200080:1.0:1787826349.772114:0:68312:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908) cmd=c0086915 arg=15793f0 00000100:00100000:1.0:1787826349.772184:0:68312:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68312:1874670011448064:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826349.772262:0:68312:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d872489c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826349.784559:0:68312:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68312:1874670011448064:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826349.784803:0:68312:0:(file.c:479:ll_file_release()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00010000:00010000:1.0:1787826349.784834:0:68312:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b9b5 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x205:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10358 expref: -99 pid: 68307 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1787826349.821697:0:68315:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826349.821723:0:68315:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826349.821739:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:1.0:1787826349.821754:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:1.0:1787826349.821762:0:68315:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826349.821769:0:68315:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826349.821772:0:68315:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:1.0:1787826349.821779:0:68315:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142b27b58 00010000:00010000:1.0:1787826349.821783:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:1.0:1787826349.821791:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:1.0:1787826349.821805:0:68315:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826349.821813:0:68315:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=25 00000080:00200000:1.0:1787826349.821819:0:68315:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:1.0:1787826349.821830:0:68315:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), flags=9 00000080:00200000:1.0:1787826349.821834:0:68315:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), intent=getattr 00000002:00010000:1.0:1787826349.821846:0:68315:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x205:0x0], intent: getattr flags 01 00010000:00010000:1.0:1787826349.821893:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.821901:0:68315:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.821911:0:68315:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826349.821952:0:68315:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011448192:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826349.822062:0:68315:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d872489c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826349.825955:0:68315:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011448192:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826349.825980:0:68315:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.825998:0:68315:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 4/1,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.826007:0:68315:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 4/1,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.826019:0:68315:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.826031:0:68315:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.826192:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.826203:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.826211:0:68315:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826349.826224:0:68315:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d908a1e00 x1874670011448192/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/1136 e 0 to 0 dl 1787826367 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826349.826235:0:68315:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826349.826247:0:68315:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.826257:0:68315:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/518 (0 0) 00000080:00200000:1.0:1787826349.826263:0:68315:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x205:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1787826349.826312:0:68315:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x206:0x0](ffff907d93315b88): rc = 0 00000080:00200000:1.0:1787826349.826329:0:68315:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x206:0x0]: layout version change: 6 -> 6 00000080:00010000:1.0:1787826349.826334:0:68315:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) for lock 0xe5f736940357b9ca 00000080:00010000:1.0:1787826349.826344:0:68315:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) 00000080:00010000:1.0:1787826349.826347:0:68315:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b9ca from it ffff9dd142b27b20 00010000:00010000:1.0:1787826349.826351:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.826359:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:1.0:1787826349.826598:0:68315:0:(file.c:6313:ll_getattr_dentry()) START file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826349.826636:0:68315:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88),name=f204d.sanity-flr 00000002:00010000:1.0:1787826349.826656:0:68315:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x206:0x0]) in obj [0x200000404:0x205:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826349.826666:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826349.826677:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826349.826685:0:68315:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x205:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826349.826689:0:68315:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) 00000080:00010000:1.0:1787826349.826692:0:68315:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) for lock 0xe5f736940357b9ca 00000080:00010000:1.0:1787826349.826696:0:68315:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b9ca from it ffff9dd142b27cb8 00010000:00010000:1.0:1787826349.826699:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826349.826707:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826349.826979:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1787826349.826989:0:68315:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826349.826992:0:68315:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826349.826996:0:68315:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x206:0x0] 00000008:00010000:1.0:1787826349.827013:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826349.827017:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d872489f0, osc lock ffff907d962f5998, flags 0x400000400001002 00000008:00010000:1.0:1787826349.827024:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826349.827027:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248a30, osc lock ffff907d962f5e78, flags 0x400000400001002 00010000:00010000:1.0:1787826349.827219:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.827229:0:68315:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->67108863] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.827241:0:68315:0:(ldlm_request.c:1139:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff907d908a1e00 x1874670011448320/t0(0) o101->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826349.827286:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1787826349.827307:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1787826349.827323:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011448320:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826349.827508:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (67108864 134217727) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1787826349.827522:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00010000:00010000:1.0:1787826349.827527:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826349.827534:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1787826349.827546:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826349.834225:0:2348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->67108863] (req 0->67108863) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.834255:0:2348:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc103eb expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.834268:0:2348:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc103eb expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.834278:0:2348:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc103eb expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826349.834292:0:2348:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc103eb expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1787826349.834303:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1787826349.834321:0:2348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc103eb expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.834338:0:2348:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc103eb expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.834346:0:2348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc103eb expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.834355:0:2348:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc103eb expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1787826349.834378:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011448320:192.168.203.104@tcp:101:lfs.0 00000080:00200000:3.0:1787826349.834567:0:68315:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 0 00000020:00010000:3.0:1787826349.834576:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da57938c8 (ffff907d884a7368/0) at cl_lock_release():245 00000020:00010000:3.0:1787826349.834579:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da57938c8 (ffff907d884a7368/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826349.834584:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d872489f0 (ffff907d884a7128/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826349.834589:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248a30 (ffff907d884a7098/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826349.834592:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da57938c8 (ffff907d884a7368/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826349.834594:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d872489f0 (ffff907d884a7368/1) at cl_lock_fini():72 00000020:00010000:3.0:1787826349.834599:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248a30 (ffff907d884a7368/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826349.834647:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/1, layout version 6 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787826349.834697:0:68315:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826349.834714:0:68315:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826349.834741:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:3.0:1787826349.834753:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:3.0:1787826349.834762:0:68315:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826349.834769:0:68315:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826349.834773:0:68315:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:3.0:1787826349.834778:0:68315:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142b27bf8 00010000:00010000:3.0:1787826349.834781:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:3.0:1787826349.834789:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:3.0:1787826349.834799:0:68315:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826349.834816:0:68315:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=25 00000080:00200000:3.0:1787826349.834822:0:68315:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:3.0:1787826349.834826:0:68315:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204d.sanity-flr, flags=265 00000080:00200000:3.0:1787826349.834830:0:68315:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88), inode mode 81a4 mask 46 00000080:00200008:3.0:1787826349.834837:0:68315:0:(file.c:1090:ll_file_open()) START file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88), flags 154402 00000002:00010000:3.0:1787826349.834851:0:68315:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x206:0x0]) in obj [0x200000404:0x205:0x0], intent: open flags 0100240114403 00010000:00010000:3.0:1787826349.834896:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.834904:0:68315:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826349.834914:0:68315:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826349.834921:0:68315:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83524780 x1874670011448576/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826349.834931:0:68315:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826349.834961:0:68315:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011448576:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826349.835245:0:68315:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d866e9600 going to sleep for 13 seconds 00000100:00100000:1.0:1787826349.839845:0:68315:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011448576:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826349.839872:0:68315:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc103f9 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.839888:0:68315:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc103f9 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.839896:0:68315:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc103f9 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.839908:0:68315:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc103f9 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.839918:0:68315:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc103f9 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.839930:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc103f9 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.839949:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc103f9 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.839957:0:68315:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc103f9 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826349.839974:0:68315:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffff907d83524780 x1874670011448576/t8589943007(8589943007) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1160 e 0 to 0 dl 1787826367 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826349.839988:0:68315:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d83524780 x1874670011448576/t8589943007(8589943007) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1160 e 0 to 0 dl 1787826367 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826349.840000:0:68315:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc103f9 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826349.840014:0:68315:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc103f9 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.840033:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826349.840045:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826349.840054:0:68315:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc103f9 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.840061:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x7918213f4fc103f9 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.840068:0:68315:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x7918213f4fc103f9 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.840095:0:68315:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:1.0:1787826349.840102:0:68315:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x205:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826349.840167:0:68315:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x206:0x0]: layout version change: 6 -> 6 00000080:00010000:1.0:1787826349.840172:0:68315:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) for lock 0xe5f736940357b9ca 00000080:00010000:1.0:1787826349.840179:0:68315:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b9ca from it ffff9dd142b27bf0 00010000:00010000:1.0:1787826349.840183:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826349.840190:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000002:00100000:1.0:1787826349.840212:0:68315:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d83524780 x1874670011448576/t8589943007(8589943007) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826367 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:3.0:1787826349.840654:0:47852:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1787826349.840662:0:47852:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7918213f4fc103f9 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.840670:0:47852:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8629c200 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826349.840674:0:47852:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7918213f4fc103f9 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.840682:0:47852:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7918213f4fc103f9 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.840689:0:47852:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 3/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc103f9 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.840708:0:47852:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1787826349.840742:0:47852:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 2/0,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc103f9 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.840748:0:47852:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787826349.840771:0:47852:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc103f9 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.840778:0:47852:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c200/0xe5f736940357b9d8 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc103f9 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826349.840794:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826349.840796:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826349.840797:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826349.840821:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011448704:192.168.203.104@tcp:103:ldlm_bl.0 00000080:00200000:1.0:1787826349.841280:0:68315:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88), inode mode 81a4 mask 4 00000080:00200000:1.0:1787826349.841291:0:68315:0:(xattr.c:760:ll_xattr_get()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88), xattr lov 00000080:00200008:1.0:1787826349.841323:0:68315:0:(file.c:6313:ll_getattr_dentry()) START file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826349.842166:0:68315:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88),name=f204d.sanity-flr 00000002:00010000:1.0:1787826349.842193:0:68315:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x206:0x0]) in obj [0x200000404:0x205:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826349.842206:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826349.842215:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1787826349.842222:0:68315:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x205:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826349.842228:0:68315:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) 00000080:00010000:1.0:1787826349.842232:0:68315:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) for lock 0xe5f736940357b9ca 00000080:00010000:1.0:1787826349.842237:0:68315:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b9ca from it ffff9dd142b27cd8 00010000:00010000:1.0:1787826349.842240:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826349.842246:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1787826349.842266:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1787826349.842276:0:68315:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826349.842280:0:68315:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826349.842283:0:68315:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x206:0x0] 00000008:00010000:1.0:1787826349.842301:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826349.842304:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248930, osc lock ffff907d962f4750, flags 0x400000400001002 00000008:00010000:1.0:1787826349.842309:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826349.842311:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248970, osc lock ffff907d962f55f0, flags 0x400000400001002 00010000:00010000:1.0:1787826349.842324:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc103eb expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826349.842823:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 67108863) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc103eb expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1787826349.842835:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826349.842838:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc103eb expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826349.842846:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc103eb expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826349.842864:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826349.842873:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (67108864 134217727) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1787826349.842880:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826349.842882:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826349.842888:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1787826349.842896:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826349.842905:0:68315:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 0 00000020:00010000:1.0:1787826349.842911:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_release():245 00000020:00010000:1.0:1787826349.842914:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826349.842916:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248930 (ffff907da5608ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826349.842920:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248970 (ffff907da56086c8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826349.842922:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826349.842924:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248930 (ffff907d884a7908/1) at cl_lock_fini():72 00000020:00010000:1.0:1787826349.842928:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248970 (ffff907d884a7908/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826349.844891:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/1, layout version 6 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:1.0:1787826349.844974:0:68315:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88) cmd=c00c66f3 arg=20d82a0 00000002:00010000:1.0:1787826349.845002:0:68315:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x206:0x0]) in obj [0x200000404:0x206:0x0], intent: open flags 05004040000002 00010000:00010000:1.0:1787826349.845067:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b9df lrc: 3/1,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.845082:0:68315:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x81000000001000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b9df lrc: 3/1,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x0 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826349.845093:0:68315:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826349.845104:0:68315:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a1e00 x1874670011448960/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826349.845114:0:68315:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b9df lrc: 3/1,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x81000000000000 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826349.845134:0:68315:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011448960:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826349.847439:0:68315:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d87248900 going to sleep for 13 seconds 00000100:00100000:1.0:1787826349.853869:0:68315:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011448960:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826349.853895:0:68315:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PW ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b9df lrc: 4/1,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc1040e expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.853908:0:68315:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b9df lrc: 4/1,0 mode: --/PW res: [0x200000404:0x206:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc1040e expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.853920:0:68315:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b9df lrc: 4/1,0 mode: PW/PW res: [0x200000404:0x206:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc1040e expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.853932:0:68315:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b9df lrc: 4/1,0 mode: PW/PW res: [0x200000404:0x206:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc1040e expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.853955:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b9df lrc: 5/1,1 mode: PW/PW res: [0x200000404:0x206:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc1040e expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.853968:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b9df lrc: 5/1,1 mode: PW/PW res: [0x200000404:0x206:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc1040e expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.853975:0:68315:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b9df lrc: 4/0,1 mode: PW/PW res: [0x200000404:0x206:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc1040e expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826349.853988:0:68315:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x600002b, status=0 req@ffff907d908a1e00 x1874670011448960/t8589943008(8589943008) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1160 e 0 to 0 dl 1787826367 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826349.854001:0:68315:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a1e00 x1874670011448960/t8589943008(8589943008) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1160 e 0 to 0 dl 1787826367 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826349.854015:0:68315:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b9df lrc: 3/0,1 mode: PW/PW res: [0x200000404:0x206:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc1040e expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.854028:0:68315:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 2 res 8589935620/518 (0 0) 00000080:00200000:1.0:1787826349.854034:0:68315:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x206:0x0] suppgids 0 0: rc 0 00000002:00100000:1.0:1787826349.854050:0:68315:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a1e00 x1874670011448960/t8589943008(8589943008) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826367 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1787826349.854058:0:68315:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) for lock 0xe5f736940357b9df 00000080:00010000:1.0:1787826349.854065:0:68315:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b9df from it ffff9dd142b27cc0 00010000:00010000:1.0:1787826349.854068:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b9df lrc: 3/0,1 mode: PW/PW res: [0x200000404:0x206:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc1040e expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.854074:0:68315:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b9df lrc: 2/0,0 mode: PW/PW res: [0x200000404:0x206:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc1040e expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200000:1.0:1787826349.854132:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 6 restore needed 0 00020000:00200000:1.0:1787826349.854154:0:68315:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826349.854158:0:68315:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826349.854164:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1787826349.854173:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:1.0:1787826349.854177:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000100:00100000:0.0:1787826349.854364:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011449088:192.168.203.104@tcp:1:lfs.0 00000100:00100000:0.0:1787826349.854458:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1787826349.854466:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011449216:192.168.203.104@tcp:1:lfs.0 00000100:00100000:2.0:1787826349.857353:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826349.857397:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826349.857474:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025507456 00000100:00100000:3.0:1787826349.857482:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025507456 00000100:00100000:3.0:1787826349.857538:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025507584 00000100:00100000:3.0:1787826349.857541:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025507584 00000100:00100000:3.0:1787826349.857557:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7154 00000100:00100000:3.0:1787826349.857564:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d911483c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:6934:x1874670025507456:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826349.857579:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc103eb expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826349.857680:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826349.857686:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d911483c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:6934:x1874670025507456:12345-192.168.203.104@tcp:104: Request processed in 122us (341us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826349.857692:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7154 00000100:00100000:3.0:1787826349.857710:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7155 00000100:00100000:3.0:1787826349.857717:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d9114b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:28560:x1874670025507584:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826349.857731:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826349.857758:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:3.0:1787826349.857762:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d9114b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:28560:x1874670025507584:12345-192.168.203.104@tcp:104: Request processed in 47us (366us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826349.857771:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7155 00010000:00010000:0.0:1787826349.857919:0:60482:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1787826349.857927:0:60482:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc103eb expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826349.857935:0:60482:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0a400 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826349.857950:0:60482:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800428400000000 nid: local remote: 0x7918213f4fc103eb expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826349.857968:0:60482:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1787826349.857980:0:60482:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1787826349.858022:0:60482:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 2/0,0 mode: --/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc103eb expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826349.858030:0:60482:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826349.858050:0:60482:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc103eb expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826349.858059:0:60482:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357b9d1 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc103eb expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826349.858079:0:60482:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826349.858083:0:60482:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826349.858089:0:60482:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a600 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826349.858093:0:60482:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800428400000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1787826349.858112:0:60482:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800429400000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1787826349.858126:0:60482:0:(osc_cache.c:1174:osc_extent_make_ready()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800429400000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826349.858388:0:60482:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (69206016 69210111) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800429400000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:0.0:1787826349.858899:0:60482:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d90846580 - object 0x280000400:782 offset 69206016<>70254592 00000100:00100000:0.0:1787826349.858950:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011449472:192.168.203.104@tcp:4:dd.0 00000100:00100000:0.0:1787826349.859031:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1787826349.859039:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011449344:192.168.203.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1787826349.860023:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826349.862441:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011448704:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826349.866680:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011449344:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826349.870684:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011449088:192.168.203.104@tcp:1:lfs.0 00000008:00010000:0.0:1787826349.915244:0:2349:0:(osc_cache.c:908:osc_extent_finish()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800429400000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00010000:0.0:1787826349.915463:0:2349:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 0000000043c02a3f ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800429400000000 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000100:00100000:0.0:1787826349.915492:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011449472:192.168.203.104@tcp:4:dd.0 00000080:00200000:3.0:1787826349.915637:0:60482:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1787826349.916076:0:60482:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:3.0:1787826349.916126:0:60482:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826349.916135:0:60482:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787826349.916176:0:60482:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826349.916185:0:60482:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357b992 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc102ef expref: -99 pid: 68279 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826349.916205:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826349.916207:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826349.916209:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826349.916218:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1787826349.916231:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011449600:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1787826349.918933:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011449600:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826349.936404:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011449216:192.168.203.104@tcp:1:lfs.0 00010000:00010000:1.0:1787826349.936604:0:68315:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1787826349.936681:0:68315:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x206:0x0]: losing layout lock 00010000:00010000:1.0:1787826349.936717:0:68315:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1787826349.936733:0:68315:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x206:0x0]: losing layout lock 00010000:00010000:1.0:1787826349.936774:0:68315:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 1/0,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826349.936783:0:68315:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 1/0,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826349.936789:0:68315:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1787826349.936793:0:68315:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9ca lrc: 0/0,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc103e4 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826349.936809:0:68315:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357b98b lrc: 0/0,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc102e8 expref: -99 pid: 68279 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1787826349.936825:0:68315:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1787826349.936836:0:68315:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a03c0 x1874670011449856/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 472/440 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826349.936862:0:68315:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011449856:192.168.203.104@tcp:36:lfs.0 00000100:00100000:1.0:1787826349.936973:0:68315:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d87248900 going to sleep for 13 seconds 00010000:00010000:2.0:1787826349.943746:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826349.969694:0:68315:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589943009 gen 1 00000100:00100000:1.0:1787826349.969719:0:68315:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d908a0780 x1874670011447168/t8589943006(8589943006) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826367 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1787826349.969743:0:68315:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d83524780 x1874670011448576/t8589943007(8589943007) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826367 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1787826349.969753:0:68315:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d908a1e00 x1874670011448960/t8589943008(8589943008) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826367 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1787826349.969765:0:68315:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011449856:192.168.203.104@tcp:36:lfs.0 00010000:00010000:1.0:1787826349.969819:0:68315:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 31 res 8589935620/518 (0 0) 00000080:00010000:1.0:1787826349.969832:0:68315:0:(file.c:7076:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000404:0x206:0x0](ffff907d93315b88) 00000002:00010000:1.0:1787826349.969845:0:68315:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x206:0x0]) in obj [0x200000404:0x206:0x0], intent: layout flags 0 00010000:00010000:1.0:1787826349.969895:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.969906:0:68315:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.969916:0:68315:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826349.969946:0:68315:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011449984:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826349.970041:0:68315:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d87248900 going to sleep for 13 seconds 00000100:00100000:1.0:1787826349.976370:0:68315:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011449984:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826349.976410:0:68315:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.976427:0:68315:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826349.976443:0:68315:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x400 disposition=0x1, status=0 req@ffff907d908a03c0 x1874670011449984/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/896 e 0 to 0 dl 1787826367 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826349.976455:0:68315:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826349.976470:0:68315:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.976486:0:68315:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/518 (0 0) 00000080:00200000:1.0:1787826349.976493:0:68315:0:(namei.c:952:ll_intent_lock()) intent lock 1024 on i1 [0x200000404:0x206:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1787826349.976523:0:68315:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) for lock 0xe5f736940357b9e6 00000080:00010000:1.0:1787826349.976532:0:68315:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b9e6 from it ffff9dd142b27c70 00010000:00010000:1.0:1787826349.976537:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.976548:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826349.976574:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826349.976592:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:1.0:1787826349.976601:0:68315:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x206:0x0](00000000407d6ff3) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826349.976692:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826349.976700:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826349.976719:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826349.976923:0:68315:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826349.976927:0:68315:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826349.976942:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1787826349.976956:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:1.0:1787826349.976998:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:1.0:1787826349.977023:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1787826349.977099:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826349.977110:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:1.0:1787826349.977119:0:68315:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x206:0x0](00000000407d6ff3) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826349.977144:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826349.977148:0:68315:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826349.977150:0:68315:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826349.977153:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1787826349.977156:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:1.0:1787826349.977196:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:1.0:1787826349.977204:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826349.977289:0:68315:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x206:0x0]: layout version change: 4294967294 -> 7 00010000:00010000:1.0:1787826349.977295:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826349.977304:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200080:1.0:1787826349.977327:0:68315:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88) cmd=66f4 arg=20d82a0 00000080:00200008:1.0:1787826349.977342:0:68315:0:(file.c:6313:ll_getattr_dentry()) START file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826349.977351:0:68315:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88),name=f204d.sanity-flr 00000002:00010000:1.0:1787826349.977364:0:68315:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x206:0x0]) in obj [0x200000404:0x206:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826349.977371:0:68315:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 30 res 8589935620/518 (0 0) 00010000:00010000:1.0:1787826349.977410:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 3/1,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.977416:0:68315:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 3/1,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.977427:0:68315:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 3/1,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826349.977460:0:68315:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011450112:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826349.978424:0:68315:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d87248900 going to sleep for 13 seconds 00000100:00100000:1.0:1787826349.984812:0:68315:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011450112:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826349.984847:0:68315:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 4/1,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.984872:0:68315:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 4/1,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.984888:0:68315:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.984901:0:68315:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.984916:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.984926:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.984934:0:68315:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826349.984956:0:68315:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d908a03c0 x1874670011450112/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/1136 e 0 to 0 dl 1787826367 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826349.984975:0:68315:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826349.984991:0:68315:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.985008:0:68315:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/518 (0 0) 00000080:00200000:1.0:1787826349.985016:0:68315:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x206:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826349.985105:0:68315:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x206:0x0]: layout version change: 7 -> 7 00000080:00010000:1.0:1787826349.985111:0:68315:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) for lock 0xe5f736940357b9ed 00000080:00010000:1.0:1787826349.985117:0:68315:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) 00000080:00010000:1.0:1787826349.985120:0:68315:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b9ed from it ffff9dd142b27cd8 00010000:00010000:1.0:1787826349.985123:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.985132:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826349.985172:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/1, layout version 6 restore needed 0 00020000:00200000:1.0:1787826349.985185:0:68315:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826349.985189:0:68315:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826349.985193:0:68315:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x206:0x0] 00000008:00010000:1.0:1787826349.985214:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826349.985218:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248930, osc lock ffff907d962f55f0, flags 0x400000400001002 00000008:00010000:1.0:1787826349.985225:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826349.985226:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248970, osc lock ffff907d962f4750, flags 0x400000400001002 00010000:00010000:1.0:1787826349.985282:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.985295:0:68315:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->67108863] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.985306:0:68315:0:(ldlm_request.c:1139:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff907d908a03c0 x1874670011450240/t0(0) o101->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826349.985355:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 3/1,0 mode: --/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.985364:0:68315:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 3/1,0 mode: --/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [67108864->134217727] (req 67108864->134217727) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826349.985374:0:68315:0:(ldlm_request.c:1139:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff907d908a1a40 x1874670011450368/t0(0) o101->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000020:00200000:1.0:1787826349.985385:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000100:00100000:0.0:1787826349.985901:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011450240:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826349.986033:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1787826349.986040:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011450368:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826349.992395:0:2348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 6/2,0 mode: --/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [67108864->134217727] (req 67108864->134217727) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826349.992430:0:2348:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 6/2,0 mode: --/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826349.992449:0:2348:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826349.992463:0:2348:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826349.992495:0:2348:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 70254592, setting rss=70254592; kms=70254592, end=18446744073709551615 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1787826349.992526:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1787826349.992532:0:2348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826349.992541:0:2348:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826349.992549:0:2348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826349.992559:0:2348:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1787826349.992583:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011450368:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826349.992603:0:2348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->67108863] (req 0->67108863) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826349.992662:0:2348:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826349.992672:0:2348:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826349.992681:0:2348:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826349.992692:0:2348:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:0.0:1787826349.992701:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1787826349.992708:0:2348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826349.992717:0:2348:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826349.992733:0:2348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826349.992741:0:2348:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1787826349.992753:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011450240:192.168.203.104@tcp:101:lfs.0 00000080:00200000:1.0:1787826349.993188:0:68315:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:1.0:1787826349.993196:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_release():245 00000020:00010000:1.0:1787826349.993200:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826349.993206:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248930 (ffff907da5608ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826349.993211:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248970 (ffff907da5608d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826349.993214:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826349.993218:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248930 (ffff907d884a7908/1) at cl_lock_fini():72 00000020:00010000:1.0:1787826349.993223:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248970 (ffff907d884a7908/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826349.993240:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:1.0:1787826350.166187:0:68315:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88) cmd=400866b1 arg=1 00000080:00200008:1.0:1787826350.166215:0:68315:0:(file.c:6313:ll_getattr_dentry()) START file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826350.166324:0:68315:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88),name=f204d.sanity-flr 00000002:00010000:1.0:1787826350.166346:0:68315:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x206:0x0]) in obj [0x200000404:0x206:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826350.166363:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826350.166377:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826350.166385:0:68315:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x206:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826350.166394:0:68315:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) 00000080:00010000:1.0:1787826350.166397:0:68315:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) for lock 0xe5f736940357b9ed 00000080:00010000:1.0:1787826350.166405:0:68315:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b9ed from it ffff9dd142b27cd8 00010000:00010000:1.0:1787826350.166409:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826350.166418:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826350.166443:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:1.0:1787826350.166454:0:68315:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826350.166457:0:68315:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826350.166461:0:68315:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x206:0x0] 00000008:00010000:1.0:1787826350.166477:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826350.166480:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248930, osc lock ffff907d962f4750, flags 0x400000400001002 00000008:00010000:1.0:1787826350.166486:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826350.166487:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248970, osc lock ffff907d962f55f0, flags 0x400000400001002 00010000:00010000:1.0:1787826350.166501:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826350.166517:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 67108863) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1787826350.166528:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826350.166531:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826350.166538:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826350.166553:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826350.166563:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (67108864 134217727) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:1.0:1787826350.166570:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826350.166573:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826350.166579:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1787826350.166589:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826350.166597:0:68315:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:1.0:1787826350.166601:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_release():245 00000020:00010000:1.0:1787826350.166604:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.166630:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248930 (ffff907da5608d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.166634:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248970 (ffff907da5608ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.166636:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.166639:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248930 (ffff907d884a7908/1) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.166643:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248970 (ffff907d884a7908/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826350.166654:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826350.166668:0:68315:0:(file.c:5224:ll_file_seek()) START file f204d.sanity-flr:[0x200000404:0x206:0x0], offset: 10485760, type: SEEK_DATA 00000080:00200000:1.0:1787826350.166673:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:1.0:1787826350.166677:0:68315:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826350.166680:0:68315:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826350.166684:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1787826350.166688:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:1.0:1787826350.166691:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:1.0:1787826350.166703:0:68315:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 2048 00000080:00200000:1.0:1787826350.166710:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826350.166715:0:68315:0:(file.c:1848:ll_io_set_mirror()) f204d.sanity-flr: desiginated mirror: 1 00000080:00200000:1.0:1787826350.166717:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826350.166721:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1787826350.166723:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:1.0:1787826350.166726:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:1.0:1787826350.166728:0:68315:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [2560, 18446744073709551615] 00000008:00010000:1.0:1787826350.166733:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:1.0:1787826350.166735:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248930, osc lock ffff907d962f55f0, flags 0x0 00000008:00010000:1.0:1787826350.166738:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:1.0:1787826350.166739:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248970, osc lock ffff907d962f4750, flags 0x0 00010000:00010000:1.0:1787826350.166747:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826350.166763:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (10485760 67108863) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826350.166772:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826350.166781:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826350.166788:0:68315:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826350.166796:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826350.166803:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (67108864 134217727) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826350.166811:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826350.166819:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826350.166831:0:68315:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826350.166839:0:68315:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 2048 00000100:00100000:0.0:1787826350.166946:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011450496:192.168.203.104@tcp:23:lfs.0 00000100:00100000:0.0:1787826350.169922:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011450496:192.168.203.104@tcp:23:lfs.0 00000020:00010000:1.0:1787826350.170018:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b17b8 (ffff907d884a7908/0) at cl_lock_release():245 00000020:00010000:1.0:1787826350.170023:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b17b8 (ffff907d884a7908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.170028:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248930 (ffff907da5608d88/1) at cl_lock_cancel():142 00010000:00010000:1.0:1787826350.170038:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826350.170054:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00010000:1.0:1787826350.170078:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248970 (ffff907da5608ea8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1787826350.170083:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826350.170091:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00010000:1.0:1787826350.170099:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b17b8 (ffff907d884a7908/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.170102:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248930 (ffff907d884a7908/1) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.170107:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248970 (ffff907d884a7908/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826350.170121:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826350.170129:0:68315:0:(file.c:5262:ll_file_seek()) COMPLETED file f204d.sanity-flr:[0x200000404:0x206:0x0], offset: 10485760, type: SEEK_DATA, rc = 69206016 00000080:00200008:1.0:1787826350.170142:0:68315:0:(file.c:5224:ll_file_seek()) START file f204d.sanity-flr:[0x200000404:0x206:0x0], offset: 69206016, type: SEEK_HOLE 00000080:00200000:1.0:1787826350.170150:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:1.0:1787826350.170168:0:68315:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826350.170171:0:68315:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826350.170176:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:1.0:1787826350.170182:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:1.0:1787826350.170197:0:68315:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 2048 00000080:00200000:1.0:1787826350.170202:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826350.170208:0:68315:0:(file.c:1848:ll_io_set_mirror()) f204d.sanity-flr: desiginated mirror: 1 00000080:00200000:1.0:1787826350.170210:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826350.170215:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:1.0:1787826350.170218:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:1.0:1787826350.170220:0:68315:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [16896, 18446744073709551615] 00000008:00010000:1.0:1787826350.170231:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:1.0:1787826350.170233:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c32630, osc lock ffff907d962f4750, flags 0x0 00010000:00010000:1.0:1787826350.170251:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826350.170263:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (69206016 134217727) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826350.170274:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826350.170282:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826350.170294:0:68315:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1787826350.170306:0:68315:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 2048 00000100:00100000:0.0:1787826350.170491:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011450624:192.168.203.104@tcp:23:lfs.0 00000100:00100000:0.0:1787826350.172511:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011450624:192.168.203.104@tcp:23:lfs.0 00000020:00010000:1.0:1787826350.172763:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b17b8 (ffff907d884a7908/0) at cl_lock_release():245 00000020:00010000:1.0:1787826350.172768:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b17b8 (ffff907d884a7908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.172772:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c32630 (ffff907da5608ea8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1787826350.172781:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826350.172796:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:1.0:1787826350.172812:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b17b8 (ffff907d884a7908/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.172815:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c32630 (ffff907d884a7908/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826350.172829:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826350.172835:0:68315:0:(file.c:5262:ll_file_seek()) COMPLETED file f204d.sanity-flr:[0x200000404:0x206:0x0], offset: 69206016, type: SEEK_HOLE, rc = 70254592 00000080:00200080:1.0:1787826350.172847:0:68315:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88) cmd=400866b1 arg=0 00000080:00200080:1.0:1787826350.172855:0:68315:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88) cmd=400866b1 arg=2 00000080:00200008:1.0:1787826350.172861:0:68315:0:(file.c:6313:ll_getattr_dentry()) START file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826350.172869:0:68315:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88),name=f204d.sanity-flr 00000002:00010000:1.0:1787826350.172887:0:68315:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x206:0x0]) in obj [0x200000404:0x206:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826350.172901:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826350.172918:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826350.172927:0:68315:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x206:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826350.172933:0:68315:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) 00000080:00010000:1.0:1787826350.172949:0:68315:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) for lock 0xe5f736940357b9ed 00000080:00010000:1.0:1787826350.172955:0:68315:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b9ed from it ffff9dd142b27cd8 00010000:00010000:1.0:1787826350.172959:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826350.172967:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1787826350.172985:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:1.0:1787826350.173215:0:68315:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826350.173219:0:68315:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826350.173224:0:68315:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x206:0x0] 00000008:00010000:1.0:1787826350.173239:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826350.173242:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248930, osc lock ffff907d962f4750, flags 0x400000400001002 00000008:00010000:1.0:1787826350.173247:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826350.173249:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248970, osc lock ffff907d962f55f0, flags 0x400000400001002 00010000:00010000:1.0:1787826350.173265:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826350.173276:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 67108863) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1787826350.173291:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826350.173295:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826350.173303:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826350.173318:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826350.173328:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (67108864 134217727) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1787826350.173335:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826350.173339:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826350.173347:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1787826350.173355:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826350.173363:0:68315:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:1.0:1787826350.173366:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_release():245 00000020:00010000:1.0:1787826350.173368:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.173371:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248930 (ffff907da5608ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.173374:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248970 (ffff907da5608d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.173377:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.173379:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248930 (ffff907d884a7908/1) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.173383:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248970 (ffff907d884a7908/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826350.173391:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826350.173427:0:68315:0:(llite_lib.c:2427:ll_setattr_raw()) START file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88) current size 70254592, valid attrs 2068, mode 0, uid -1976721614, gid -1, new size 10485760, atime 1787826350.000000000, mtime 1787826350.000000000, ctime 1787826350.000000000 00010000:00010000:1.0:1787826350.173446:0:68315:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826350.173463:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826350.173498:0:68315:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 1/0,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826350.173505:0:68315:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826350.173509:0:68315:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a400/0xe5f736940357b9ed lrc: 0/0,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc1043f expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1787826350.174244:0:68315:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1787826350.174258:0:68315:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a21c0 x1874670011450880/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826350.174282:0:68315:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011450880:192.168.203.104@tcp:36:lfs.0 00000100:00100000:1.0:1787826350.174424:0:68315:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d87248900 going to sleep for 13 seconds 00000100:00100000:1.0:1787826350.193953:0:68315:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011450880:192.168.203.104@tcp:36:lfs.0 00000080:00200000:1.0:1787826350.193996:0:68315:0:(file.c:1848:ll_io_set_mirror()) f204d.sanity-flr: desiginated mirror: 2 00000080:00200000:1.0:1787826350.194003:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:1.0:1787826350.194024:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[4] flags 0x11 00000080:00200000:1.0:1787826350.194040:0:68315:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1787826350.194057:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1787826350.194062:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248930, osc lock ffff907d962f55f0, flags 0x0 00000008:00010000:1.0:1787826350.194226:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1787826350.194229:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248970, osc lock ffff907d962f4750, flags 0x0 00010000:00010000:1.0:1787826350.194285:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.194298:0:68315:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.194308:0:68315:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826350.194327:0:68315:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011451008:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826350.194412:0:68315:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d872489c0 going to sleep for 11 seconds 00000100:00100000:1.0:1787826350.216573:0:68315:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011451008:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826350.216600:0:68315:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.216654:0:68315:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.216665:0:68315:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.216685:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826350.216697:0:68315:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.216708:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.216716:0:68315:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.216789:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357ba09 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.216798:0:68315:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357ba09 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [10485760->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.216807:0:68315:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357ba09 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [10485760->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826350.216827:0:68315:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011451136:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826350.216913:0:68315:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d872489c0 going to sleep for 11 seconds 00000100:00100000:1.0:1787826350.222078:0:68315:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011451136:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826350.222105:0:68315:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357ba09 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10462 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.222125:0:68315:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357ba09 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10462 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.222138:0:68315:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357ba09 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10462 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.222845:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357ba09 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10462 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826350.222866:0:68315:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357ba09 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10462 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.222880:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357ba09 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10462 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.222889:0:68315:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357ba09 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10462 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826350.223042:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1787826350.223059:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011451264:192.168.203.104@tcp:10:lfs.0 00000100:00100000:0.0:1787826350.235074:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011451264:192.168.203.104@tcp:10:lfs.0 00000080:00200000:1.0:1787826350.235560:0:68315:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:1.0:1787826350.235568:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b17b8 (ffff907d884a7908/0) at cl_lock_release():245 00000020:00010000:1.0:1787826350.235571:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b17b8 (ffff907d884a7908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.235576:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248930 (ffff907da5608ea8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1787826350.235588:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.235603:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:1.0:1787826350.235640:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248970 (ffff907da5608d88/1) at cl_lock_cancel():142 00010000:00010000:1.0:1787826350.235645:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357ba09 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10462 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.235655:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357ba09 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10462 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:1.0:1787826350.235664:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b17b8 (ffff907d884a7908/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.235667:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248930 (ffff907d884a7908/1) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.235673:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248970 (ffff907d884a7908/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826350.235692:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826350.235704:0:68315:0:(llite_lib.c:2686:ll_setattr_raw()) COMPLETED file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88) current size 70254592, valid attrs 10002168, mode 0, uid -1976721614, gid -1, new size 10485760, atime 1787826350.000000000, mtime 1787826350.000000000, ctime 1787826350.000000000, rc 0 00000080:00200080:1.0:1787826350.235721:0:68315:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88) cmd=400866b1 arg=0 00000080:00200080:1.0:1787826350.235731:0:68315:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88) cmd=400866b1 arg=1 00000080:00200008:1.0:1787826350.235737:0:68315:0:(file.c:6313:ll_getattr_dentry()) START file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826350.235744:0:68315:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88),name=f204d.sanity-flr 00000002:00010000:1.0:1787826350.235759:0:68315:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x206:0x0]) in obj [0x200000404:0x206:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826350.235772:0:68315:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 30 res 8589935620/518 (0 0) 00010000:00010000:1.0:1787826350.235817:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.235826:0:68315:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.235836:0:68315:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826350.235856:0:68315:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011451392:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826350.235953:0:68315:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d87248900 going to sleep for 13 seconds 00000100:00100000:1.0:1787826350.251087:0:68315:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011451392:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826350.251113:0:68315:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.251126:0:68315:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.251137:0:68315:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.251148:0:68315:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.251160:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.251167:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.251180:0:68315:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826350.251192:0:68315:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d908a0000 x1874670011451392/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/1136 e 0 to 0 dl 1787826368 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826350.251204:0:68315:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826350.251217:0:68315:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.251229:0:68315:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/518 (0 0) 00000080:00200000:1.0:1787826350.251235:0:68315:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x206:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826350.251299:0:68315:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x206:0x0]: layout version change: 7 -> 7 00000080:00010000:1.0:1787826350.251304:0:68315:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) for lock 0xe5f736940357ba10 00000080:00010000:1.0:1787826350.251309:0:68315:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) 00000080:00010000:1.0:1787826350.251312:0:68315:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba10 from it ffff9dd142b27cd8 00010000:00010000:1.0:1787826350.251316:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.251324:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826350.251353:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:1.0:1787826350.251363:0:68315:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826350.251366:0:68315:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826350.251370:0:68315:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x206:0x0] 00000008:00010000:1.0:1787826350.251386:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826350.251389:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248930, osc lock ffff907d962f4750, flags 0x400000400001002 00000008:00010000:1.0:1787826350.251396:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826350.251398:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248970, osc lock ffff907d962f55f0, flags 0x400000400001002 00010000:00010000:1.0:1787826350.251413:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826350.251426:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 67108863) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1787826350.251438:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826350.251441:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826350.251449:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826350.251465:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826350.251473:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (67108864 134217727) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1787826350.251480:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826350.251483:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826350.251495:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000020:00200000:1.0:1787826350.251503:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826350.251522:0:68315:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:1.0:1787826350.252077:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_release():245 00000020:00010000:1.0:1787826350.252082:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.252086:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248930 (ffff907da5608ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.252090:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248970 (ffff907da5608d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.252092:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.252095:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248930 (ffff907d884a7908/1) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.252099:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248970 (ffff907d884a7908/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826350.252109:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826350.252139:0:68315:0:(file.c:2485:do_file_read_iter()) START file f204d.sanity-flr:[0x200000404:0x206:0x0], ppos: 69206016, count: 1048576 00000080:00200000:1.0:1787826350.252149:0:68315:0:(file.c:2090:ll_file_io_generic()) f204d.sanity-flr: read ppos: 69206016, bytes: 1048576 00000080:00200000:1.0:1787826350.252156:0:68315:0:(file.c:1848:ll_io_set_mirror()) f204d.sanity-flr: desiginated mirror: 1 00000020:00200000:1.0:1787826350.252163:0:68315:0:(cl_io.c:218:cl_io_rw_init()) header@000000008ae07a1f[0x110, 2, [0x200000404:0x206:0x0] hash] 00000020:00200000:1.0:1787826350.252165:0:68315:0:(cl_io.c:218:cl_io_rw_init()) io range: 1 [69206016, 70254592) 2048 0 00000080:00200000:1.0:1787826350.252168:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00000080:00200000:1.0:1787826350.252177:0:68315:0:(file.c:2170:ll_file_io_generic()) Range lock [16896, 17151] 00020000:00200000:1.0:1787826350.252183:0:68315:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 69206016 chunk: [69206016, 70254592) 70254592, 1048576 00020000:00200000:1.0:1787826350.252187:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00000080:00200000:1.0:1787826350.252198:0:68315:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [16896, 17151] 00000008:00010000:1.0:1787826350.252207:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 11 00000008:00010000:1.0:1787826350.252210:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c32630, osc lock ffff907d962f55f0, flags 0x40000 00000080:00200000:1.0:1787826350.252215:0:68315:0:(vvp_io.c:874:vvp_io_read_start()) f204d.sanity-flr: read [69206016, 70254592) 00000080:00200000:1.0:1787826350.252221:0:68315:0:(vvp_io.c:194:vvp_prep_size()) [0x200000404:0x206:0x0] updating i_size 70254592 00000080:00200000:1.0:1787826350.252228:0:68315:0:(vvp_io.c:932:vvp_io_read_start()) count:1048576 iocb pos:69206016 00000080:00200000:1.0:1787826350.252233:0:68315:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88), size=1048576 (max 335544320), offset=69206016=0x4200000, pages 256 (max 81920), lockless, parallel 00000020:00200000:1.0:1787826350.255074:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:1.0:1787826350.255084:0:68315:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 70254592 00000020:00010000:1.0:1787826350.255093:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b17b8 (ffff907d884a7908/0) at cl_lock_release():245 00000020:00010000:1.0:1787826350.255097:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b17b8 (ffff907d884a7908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.255100:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c32630 (ffff907da5608d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.255104:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b17b8 (ffff907d884a7908/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.255106:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c32630 (ffff907d884a7908/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826350.255116:0:68315:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:1.0:1787826350.255123:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1787826350.255148:0:58785:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907d908aac00 00000008:00100000:1.0:1787826350.255253:0:58785:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d908a0000 - object 0x280000400:782 offset 69206016<>70254592 00000100:00100000:0.0:1787826350.255414:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011451520:192.168.203.104@tcp:3:lfs.0 00000008:00010000:0.0:1787826350.294598:0:2349:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907d908aac00 00000020:00200000:0.0:1787826350.294643:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:0.0:1787826350.294647:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:0.0:1787826350.294648:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:0.0:1787826350.294650:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:0.0:1787826350.294652:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:0.0:1787826350.294654:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:0.0:1787826350.294656:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:0.0:1787826350.294657:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:0.0:1787826350.294659:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:0.0:1787826350.294661:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:0.0:1787826350.294663:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:0.0:1787826350.294664:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:0.0:1787826350.294666:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:0.0:1787826350.294667:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:0.0:1787826350.294669:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:0.0:1787826350.294670:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:0.0:1787826350.294672:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:0.0:1787826350.294674:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:0.0:1787826350.294675:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:0.0:1787826350.294677:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:0.0:1787826350.294678:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:0.0:1787826350.294680:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:0.0:1787826350.294694:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:0.0:1787826350.294697:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:0.0:1787826350.294699:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:0.0:1787826350.294702:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:0.0:1787826350.294703:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:0.0:1787826350.294705:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:0.0:1787826350.294707:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:0.0:1787826350.294709:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:0.0:1787826350.294710:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:0.0:1787826350.294712:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:0.0:1787826350.294714:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:0.0:1787826350.294715:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:0.0:1787826350.294717:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:0.0:1787826350.294718:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:0.0:1787826350.294720:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:0.0:1787826350.294721:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:0.0:1787826350.294923:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:0.0:1787826350.294928:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:0.0:1787826350.294930:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:0.0:1787826350.294931:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:0.0:1787826350.294932:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:0.0:1787826350.294942:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:0.0:1787826350.294943:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:0.0:1787826350.294945:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:0.0:1787826350.294947:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:0.0:1787826350.294948:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:0.0:1787826350.294950:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:0.0:1787826350.294951:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:0.0:1787826350.294953:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:0.0:1787826350.294954:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:0.0:1787826350.294956:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:0.0:1787826350.294957:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:0.0:1787826350.294959:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:0.0:1787826350.294960:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:0.0:1787826350.294961:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:0.0:1787826350.294963:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:0.0:1787826350.294965:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:0.0:1787826350.294967:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:0.0:1787826350.294968:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:0.0:1787826350.294970:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:0.0:1787826350.294971:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:0.0:1787826350.294972:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:0.0:1787826350.294974:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:0.0:1787826350.294975:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:0.0:1787826350.294977:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:0.0:1787826350.294978:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:0.0:1787826350.294980:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:0.0:1787826350.294981:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:0.0:1787826350.294983:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:0.0:1787826350.294984:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:0.0:1787826350.294985:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:0.0:1787826350.294987:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:0.0:1787826350.294988:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:0.0:1787826350.294990:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:0.0:1787826350.294991:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:0.0:1787826350.294993:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:0.0:1787826350.294994:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:0.0:1787826350.294996:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:0.0:1787826350.294997:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:0.0:1787826350.294999:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:0.0:1787826350.295001:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:0.0:1787826350.295002:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:0.0:1787826350.295004:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:0.0:1787826350.295005:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:0.0:1787826350.295007:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:0.0:1787826350.295008:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:0.0:1787826350.295010:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:0.0:1787826350.295012:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:0.0:1787826350.295014:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:0.0:1787826350.295015:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:0.0:1787826350.295017:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:0.0:1787826350.295018:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:0.0:1787826350.295020:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:0.0:1787826350.295022:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:0.0:1787826350.295023:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:0.0:1787826350.295024:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:0.0:1787826350.295026:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:0.0:1787826350.295028:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:0.0:1787826350.295030:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:0.0:1787826350.295032:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:0.0:1787826350.295034:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:0.0:1787826350.295035:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:0.0:1787826350.295037:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:0.0:1787826350.295039:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:0.0:1787826350.295041:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:0.0:1787826350.295042:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:0.0:1787826350.295044:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:0.0:1787826350.295046:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:0.0:1787826350.295047:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:0.0:1787826350.295049:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:0.0:1787826350.295050:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:0.0:1787826350.295051:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:0.0:1787826350.295053:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:0.0:1787826350.295055:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:0.0:1787826350.295056:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:0.0:1787826350.295058:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:0.0:1787826350.295059:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:0.0:1787826350.295061:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:0.0:1787826350.295062:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:0.0:1787826350.295064:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:0.0:1787826350.295066:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:0.0:1787826350.295067:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:0.0:1787826350.295069:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:0.0:1787826350.295071:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:0.0:1787826350.295072:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:0.0:1787826350.295074:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:0.0:1787826350.295076:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:0.0:1787826350.295077:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:0.0:1787826350.295079:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:0.0:1787826350.295080:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:0.0:1787826350.295082:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:0.0:1787826350.295084:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:0.0:1787826350.295085:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:0.0:1787826350.295087:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:0.0:1787826350.295088:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:0.0:1787826350.295090:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:0.0:1787826350.295091:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:0.0:1787826350.295093:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:0.0:1787826350.295094:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:0.0:1787826350.295096:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:0.0:1787826350.295097:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:0.0:1787826350.295098:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:0.0:1787826350.295100:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:0.0:1787826350.295102:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:0.0:1787826350.295103:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:0.0:1787826350.295105:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:0.0:1787826350.295106:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:0.0:1787826350.295108:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:0.0:1787826350.295109:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:0.0:1787826350.295111:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:0.0:1787826350.295112:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:0.0:1787826350.295114:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:0.0:1787826350.295116:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:0.0:1787826350.295117:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:0.0:1787826350.295119:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:0.0:1787826350.295120:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:0.0:1787826350.295122:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:0.0:1787826350.295123:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:0.0:1787826350.295125:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:0.0:1787826350.295126:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:0.0:1787826350.295128:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:0.0:1787826350.295130:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:0.0:1787826350.295131:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:0.0:1787826350.295133:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:0.0:1787826350.295135:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:0.0:1787826350.295136:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:0.0:1787826350.295138:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:0.0:1787826350.295140:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:0.0:1787826350.295141:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:0.0:1787826350.295143:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:0.0:1787826350.295145:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:0.0:1787826350.295146:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:0.0:1787826350.295148:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:0.0:1787826350.295149:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:0.0:1787826350.295151:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:0.0:1787826350.295152:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:0.0:1787826350.295153:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:0.0:1787826350.295155:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:0.0:1787826350.295156:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:0.0:1787826350.295158:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:0.0:1787826350.295160:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:0.0:1787826350.295161:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:0.0:1787826350.295163:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:0.0:1787826350.295165:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:0.0:1787826350.295167:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:0.0:1787826350.295168:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:0.0:1787826350.295169:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:0.0:1787826350.295171:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:0.0:1787826350.295172:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:0.0:1787826350.295174:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:0.0:1787826350.295175:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:0.0:1787826350.295176:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:0.0:1787826350.295178:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:0.0:1787826350.295179:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:0.0:1787826350.295181:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:0.0:1787826350.295182:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:0.0:1787826350.295184:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:0.0:1787826350.295185:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:0.0:1787826350.295187:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:0.0:1787826350.295188:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:0.0:1787826350.295190:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:0.0:1787826350.295191:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:0.0:1787826350.295193:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:0.0:1787826350.295194:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:0.0:1787826350.295196:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:0.0:1787826350.295197:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:0.0:1787826350.295198:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:0.0:1787826350.295201:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:0.0:1787826350.295202:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:0.0:1787826350.295204:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:0.0:1787826350.295206:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:0.0:1787826350.295207:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:0.0:1787826350.295209:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:0.0:1787826350.295211:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:0.0:1787826350.295212:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:0.0:1787826350.295214:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:0.0:1787826350.295216:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:0.0:1787826350.295218:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:0.0:1787826350.295220:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:0.0:1787826350.295221:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:0.0:1787826350.295222:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:0.0:1787826350.295224:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:0.0:1787826350.295226:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:0.0:1787826350.295228:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:0.0:1787826350.295229:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:0.0:1787826350.295231:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:0.0:1787826350.295233:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:0.0:1787826350.295234:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:0.0:1787826350.295236:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:0.0:1787826350.295237:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:0.0:1787826350.295239:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:0.0:1787826350.295240:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:0.0:1787826350.295242:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:0.0:1787826350.295243:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:0.0:1787826350.295245:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:0.0:1787826350.295247:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:0.0:1787826350.295248:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:0.0:1787826350.295250:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:0.0:1787826350.295251:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:0.0:1787826350.295253:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:0.0:1787826350.295255:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:0.0:1787826350.295257:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:0.0:1787826350.295258:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:0.0:1787826350.295260:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:0.0:1787826350.295262:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:0.0:1787826350.295264:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:0.0:1787826350.295266:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:0.0:1787826350.295268:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:0.0:1787826350.295270:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:0.0:1787826350.295271:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:0.0:1787826350.295273:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:0.0:1787826350.295275:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:0.0:1787826350.295276:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:0.0:1787826350.295278:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1787826350.295367:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1787826350.295392:0:2349:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907d908aac00 00000100:00100000:0.0:1787826350.295418:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011451520:192.168.203.104@tcp:3:lfs.0 00000080:00200000:1.0:1787826350.297949:0:68315:0:(file.c:2207:ll_file_io_generic()) Range unlock [16896, 17151] 00000080:00200000:1.0:1787826350.297970:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826350.297975:0:68315:0:(file.c:2240:ll_file_io_generic()) f204d.sanity-flr: 1 io complete with rc: 0, result: 1048576, restart: 0 00000020:00200000:1.0:1787826350.297979:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826350.297992:0:68315:0:(file.c:2326:ll_file_io_generic()) iot: 1, result: 1048576 00000080:00200080:1.0:1787826350.298009:0:68315:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88) cmd=400866b1 arg=0 00000080:00200080:1.0:1787826350.298019:0:68315:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88) cmd=400866b1 arg=2 00000080:00200008:1.0:1787826350.298025:0:68315:0:(file.c:6313:ll_getattr_dentry()) START file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826350.298032:0:68315:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88),name=f204d.sanity-flr 00000002:00010000:1.0:1787826350.298051:0:68315:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x206:0x0]) in obj [0x200000404:0x206:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826350.298067:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826350.298081:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826350.298093:0:68315:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x206:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826350.298101:0:68315:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) 00000080:00010000:1.0:1787826350.298104:0:68315:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) for lock 0xe5f736940357ba10 00000080:00010000:1.0:1787826350.298111:0:68315:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba10 from it ffff9dd142b27cd8 00010000:00010000:1.0:1787826350.298115:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826350.298123:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826350.298148:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:1.0:1787826350.298159:0:68315:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826350.298163:0:68315:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826350.298166:0:68315:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x206:0x0] 00000008:00010000:1.0:1787826350.298186:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826350.298190:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248930, osc lock ffff907d962f55f0, flags 0x400000400001002 00000008:00010000:1.0:1787826350.298197:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826350.298198:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248970, osc lock ffff907d962f4750, flags 0x400000400001002 00010000:00010000:1.0:1787826350.298213:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826350.298224:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 67108863) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1787826350.298234:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826350.298237:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826350.298245:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826350.298260:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1787826350.298269:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (67108864 134217727) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000020:00200000:1.0:1787826350.298277:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826350.298279:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1787826350.298286:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000020:00200000:1.0:1787826350.298295:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826350.298303:0:68315:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:1.0:1787826350.298307:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_release():245 00000020:00010000:1.0:1787826350.298310:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.298313:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248930 (ffff907da5608d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.298317:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248970 (ffff907da5608ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.298320:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.298323:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248930 (ffff907d884a7908/1) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.298327:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248970 (ffff907d884a7908/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826350.298334:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826350.298353:0:68315:0:(file.c:2652:do_file_write_iter()) START file f204d.sanity-flr:[0x200000404:0x206:0x0], ppos: 69206016, count: 1048576 00000080:00200000:1.0:1787826350.298358:0:68315:0:(file.c:2090:ll_file_io_generic()) f204d.sanity-flr: write ppos: 69206016, bytes: 1048576 00000080:00200000:1.0:1787826350.298365:0:68315:0:(file.c:1848:ll_io_set_mirror()) f204d.sanity-flr: desiginated mirror: 2 00000020:00200000:1.0:1787826350.298371:0:68315:0:(cl_io.c:218:cl_io_rw_init()) header@000000008ae07a1f[0x110, 2, [0x200000404:0x206:0x0] hash] 00000020:00200000:1.0:1787826350.298373:0:68315:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [69206016, 70254592) 2048 0 00000080:00200000:1.0:1787826350.298375:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00000080:00200000:1.0:1787826350.298385:0:68315:0:(file.c:2170:ll_file_io_generic()) Range lock [16896, 17151] 00020000:00200000:1.0:1787826350.298391:0:68315:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 69206016 chunk: [69206016, 70254592) 70254592, 1048576 00020000:00200000:1.0:1787826350.298395:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[4] flags 0x11 00000080:00200000:1.0:1787826350.298408:0:68315:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [16896, 17151] 00000008:00010000:1.0:1787826350.298416:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 11 00000008:00010000:1.0:1787826350.298419:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c32630, osc lock ffff907d962f4750, flags 0x40000 00000080:00200000:1.0:1787826350.298426:0:68315:0:(vvp_io.c:1339:vvp_io_write_start()) f204d.sanity-flr: write [69206016, 70254592) append 0 append lockpos 0 00000080:00200000:1.0:1787826350.298447:0:68315:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88), size=1048576 (max 335544320), offset=69206016=0x4200000, pages 256 (max 81920), lockless, parallel 00000020:00200000:1.0:1787826350.300714:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:1.0:1787826350.300721:0:68315:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 70254592 00000080:00200000:1.0:1787826350.303963:0:68315:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:1.0:1787826350.303976:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b17b8 (ffff907d884a7908/0) at cl_lock_release():245 00000020:00010000:1.0:1787826350.303981:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b17b8 (ffff907d884a7908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.303984:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c32630 (ffff907da5608ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.303988:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b17b8 (ffff907d884a7908/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.303991:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c32630 (ffff907d884a7908/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826350.304001:0:68315:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:1.0:1787826350.304009:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1787826350.304038:0:58785:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907d908aa000 00000008:00100000:1.0:1787826350.304527:0:58785:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d908a03c0 - object 0x240000400:808 offset 69206016<>70254592 00000100:00100000:0.0:1787826350.309323:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011451648:192.168.203.104@tcp:4:lfs.0 00000100:00100000:2.0:1787826350.316803:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826350.317853:0:50445:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:3.0:1787826350.317875:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025508224 00000100:00100000:3.0:1787826350.317884:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025508224 00000100:00100000:3.0:1787826350.317952:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7156 00000100:00100000:3.0:1787826350.317962:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d84e77050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:28337:x1874670025508224:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826350.317979:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357ba09 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc10462 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826350.318069:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826350.318076:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d84e77050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:28337:x1874670025508224:12345-192.168.203.104@tcp:104: Request processed in 116us (1282us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826350.318083:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7156 00010000:00010000:0.0:1787826350.318526:0:60472:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826350.318534:0:60472:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357ba09 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc10462 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826350.318545:0:60472:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0b400 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826350.318551:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357ba09 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x7918213f4fc10462 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826350.318573:0:60472:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1787826350.318596:0:60472:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1787826350.318662:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357ba09 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc10462 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826350.318670:0:60472:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826350.318689:0:60472:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357ba09 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc10462 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826350.318703:0:60472:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357ba09 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc10462 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826350.318731:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826350.318733:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826350.318735:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826350.318747:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1787826350.318761:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011451776:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826350.324051:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011451776:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826350.366485:0:2348:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff907d986d0000: committing for last_committed 21474840511 gen 3 00000008:00010000:0.0:1787826350.366556:0:2348:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907d908aa000 00000020:00200000:0.0:1787826350.366563:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:0.0:1787826350.366566:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:0.0:1787826350.366568:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:0.0:1787826350.366569:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:0.0:1787826350.366571:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:0.0:1787826350.366573:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:0.0:1787826350.366574:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:0.0:1787826350.366576:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:0.0:1787826350.366578:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:0.0:1787826350.366580:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:0.0:1787826350.366581:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:0.0:1787826350.366593:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:0.0:1787826350.366595:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:0.0:1787826350.366596:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:0.0:1787826350.366598:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:0.0:1787826350.366600:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:0.0:1787826350.366602:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:0.0:1787826350.366603:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:0.0:1787826350.366639:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:0.0:1787826350.366642:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:0.0:1787826350.366644:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:0.0:1787826350.366645:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:0.0:1787826350.366646:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:0.0:1787826350.366648:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:0.0:1787826350.366649:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:0.0:1787826350.366651:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:0.0:1787826350.366652:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:0.0:1787826350.366654:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:0.0:1787826350.366655:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:0.0:1787826350.366657:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:0.0:1787826350.366658:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:0.0:1787826350.366660:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:0.0:1787826350.366661:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:0.0:1787826350.366664:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:0.0:1787826350.366665:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:0.0:1787826350.366667:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:0.0:1787826350.366668:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:0.0:1787826350.366670:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:0.0:1787826350.366671:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:0.0:1787826350.366673:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:0.0:1787826350.366675:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:0.0:1787826350.366677:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:0.0:1787826350.366678:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:0.0:1787826350.366680:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:0.0:1787826350.366681:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:0.0:1787826350.366834:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:0.0:1787826350.366836:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:0.0:1787826350.366838:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:0.0:1787826350.366840:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:0.0:1787826350.366842:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:0.0:1787826350.366843:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:0.0:1787826350.366845:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:0.0:1787826350.366847:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:0.0:1787826350.366849:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:0.0:1787826350.366850:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:0.0:1787826350.366852:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:0.0:1787826350.366853:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:0.0:1787826350.366855:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:0.0:1787826350.366857:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:0.0:1787826350.366859:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:0.0:1787826350.366861:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:0.0:1787826350.366862:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:0.0:1787826350.366864:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:0.0:1787826350.366866:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:0.0:1787826350.366868:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:0.0:1787826350.366870:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:0.0:1787826350.366872:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:0.0:1787826350.366873:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:0.0:1787826350.366875:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:0.0:1787826350.366877:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:0.0:1787826350.366878:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:0.0:1787826350.366880:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:0.0:1787826350.366881:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:0.0:1787826350.366883:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:0.0:1787826350.366885:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:0.0:1787826350.366887:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:0.0:1787826350.366888:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:0.0:1787826350.366890:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:0.0:1787826350.366891:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:0.0:1787826350.366894:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:0.0:1787826350.366895:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:0.0:1787826350.366898:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:0.0:1787826350.366899:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:0.0:1787826350.366901:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:0.0:1787826350.366903:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:0.0:1787826350.366904:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:0.0:1787826350.366906:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:0.0:1787826350.366908:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:0.0:1787826350.366909:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:0.0:1787826350.366911:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:0.0:1787826350.366913:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:0.0:1787826350.366914:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:0.0:1787826350.366916:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:0.0:1787826350.366918:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:0.0:1787826350.366920:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:0.0:1787826350.366922:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:0.0:1787826350.366924:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:0.0:1787826350.366926:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:0.0:1787826350.366928:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:0.0:1787826350.366930:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:0.0:1787826350.366932:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:0.0:1787826350.366944:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:0.0:1787826350.366946:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:0.0:1787826350.366948:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:0.0:1787826350.366949:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:0.0:1787826350.366950:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:0.0:1787826350.366952:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:0.0:1787826350.366954:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:0.0:1787826350.366955:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:0.0:1787826350.366957:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:0.0:1787826350.366959:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:0.0:1787826350.366961:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:0.0:1787826350.366962:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:0.0:1787826350.366964:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:0.0:1787826350.366966:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:0.0:1787826350.366968:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:0.0:1787826350.366969:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:0.0:1787826350.366971:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:0.0:1787826350.366972:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:0.0:1787826350.366974:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:0.0:1787826350.366975:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:0.0:1787826350.366977:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:0.0:1787826350.366978:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:0.0:1787826350.366980:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:0.0:1787826350.366981:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:0.0:1787826350.366983:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:0.0:1787826350.366984:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:0.0:1787826350.366986:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:0.0:1787826350.366988:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:0.0:1787826350.366989:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:0.0:1787826350.366991:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:0.0:1787826350.366993:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:0.0:1787826350.366995:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:0.0:1787826350.366996:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:0.0:1787826350.366998:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:0.0:1787826350.367000:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:0.0:1787826350.367002:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:0.0:1787826350.367003:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:0.0:1787826350.367005:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:0.0:1787826350.367006:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:0.0:1787826350.367008:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:0.0:1787826350.367010:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:0.0:1787826350.367011:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:0.0:1787826350.367014:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:0.0:1787826350.367015:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:0.0:1787826350.367017:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:0.0:1787826350.367018:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:0.0:1787826350.367020:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:0.0:1787826350.367021:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:0.0:1787826350.367023:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:0.0:1787826350.367025:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:0.0:1787826350.367026:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:0.0:1787826350.367028:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:0.0:1787826350.367030:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:0.0:1787826350.367031:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:0.0:1787826350.367033:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:0.0:1787826350.367035:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:0.0:1787826350.367037:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:0.0:1787826350.367039:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:0.0:1787826350.367040:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:0.0:1787826350.367042:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:0.0:1787826350.367044:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:0.0:1787826350.367045:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:0.0:1787826350.367047:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:0.0:1787826350.367049:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:0.0:1787826350.367050:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:0.0:1787826350.367052:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:0.0:1787826350.367053:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:0.0:1787826350.367055:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:0.0:1787826350.367057:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:0.0:1787826350.367058:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:0.0:1787826350.367060:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:0.0:1787826350.367062:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:0.0:1787826350.367063:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:0.0:1787826350.367065:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:0.0:1787826350.367067:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:0.0:1787826350.367069:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:0.0:1787826350.367071:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:0.0:1787826350.367073:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:0.0:1787826350.367074:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:0.0:1787826350.367076:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:0.0:1787826350.367077:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:0.0:1787826350.367079:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:0.0:1787826350.367081:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:0.0:1787826350.367082:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:0.0:1787826350.367084:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:0.0:1787826350.367086:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:0.0:1787826350.367088:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:0.0:1787826350.367090:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:0.0:1787826350.367091:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:0.0:1787826350.367093:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:0.0:1787826350.367094:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:0.0:1787826350.367096:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:0.0:1787826350.367098:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:0.0:1787826350.367100:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:0.0:1787826350.367102:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:0.0:1787826350.367103:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:0.0:1787826350.367105:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:0.0:1787826350.367107:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:0.0:1787826350.367109:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:0.0:1787826350.367111:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:0.0:1787826350.367113:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:0.0:1787826350.367114:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:0.0:1787826350.367116:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:0.0:1787826350.367118:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:0.0:1787826350.367119:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:0.0:1787826350.367121:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:0.0:1787826350.367123:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:0.0:1787826350.367125:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:0.0:1787826350.367126:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:0.0:1787826350.367128:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:0.0:1787826350.367130:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:0.0:1787826350.367131:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:0.0:1787826350.367133:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:0.0:1787826350.367134:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:0.0:1787826350.367136:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:0.0:1787826350.367138:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:0.0:1787826350.367139:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:0.0:1787826350.367141:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:0.0:1787826350.367143:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:0.0:1787826350.367145:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:0.0:1787826350.367147:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:0.0:1787826350.367148:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:0.0:1787826350.367150:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:0.0:1787826350.367151:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:0.0:1787826350.367153:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:0.0:1787826350.367155:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:0.0:1787826350.367156:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:0.0:1787826350.367157:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:0.0:1787826350.367159:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:0.0:1787826350.367160:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:0.0:1787826350.367162:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:0.0:1787826350.367163:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:0.0:1787826350.367165:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:0.0:1787826350.367166:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:0.0:1787826350.367168:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:0.0:1787826350.367170:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:0.0:1787826350.367172:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:0.0:1787826350.367174:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:0.0:1787826350.367176:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:0.0:1787826350.367178:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:0.0:1787826350.367179:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:0.0:1787826350.367181:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:0.0:1787826350.367183:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:0.0:1787826350.367184:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:0.0:1787826350.367186:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:0.0:1787826350.367187:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:0.0:1787826350.367189:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:0.0:1787826350.367191:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:0.0:1787826350.367193:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:0.0:1787826350.367194:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:0.0:1787826350.367196:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:0.0:1787826350.367197:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:0.0:1787826350.367199:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:0.0:1787826350.367200:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:0.0:1787826350.367202:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1787826350.367409:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1787826350.367477:0:2348:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907d908aa000 00000100:00100000:0.0:1787826350.367503:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011451648:192.168.203.104@tcp:4:lfs.0 00000080:00200000:1.0:1787826350.370791:0:68315:0:(file.c:2207:ll_file_io_generic()) Range unlock [16896, 17151] 00000080:00200000:1.0:1787826350.370810:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826350.370818:0:68315:0:(file.c:2240:ll_file_io_generic()) f204d.sanity-flr: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:1.0:1787826350.370842:0:68315:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 2048 00000020:00200000:1.0:1787826350.370847:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826350.370859:0:68315:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200080:1.0:1787826350.370876:0:68315:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88) cmd=400866b1 arg=0 00000080:00200080:1.0:1787826350.370885:0:68315:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88) cmd=400866b1 arg=1 00000080:00200008:1.0:1787826350.370891:0:68315:0:(file.c:6313:ll_getattr_dentry()) START file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826350.370900:0:68315:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88),name=f204d.sanity-flr 00000002:00010000:1.0:1787826350.370919:0:68315:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x206:0x0]) in obj [0x200000404:0x206:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826350.370945:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826350.370959:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826350.370968:0:68315:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x206:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826350.370975:0:68315:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) 00000080:00010000:1.0:1787826350.370979:0:68315:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) for lock 0xe5f736940357ba10 00000080:00010000:1.0:1787826350.370987:0:68315:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba10 from it ffff9dd142b27cd8 00010000:00010000:1.0:1787826350.370991:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826350.371000:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1787826350.371022:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:1.0:1787826350.371033:0:68315:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826350.371036:0:68315:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826350.371041:0:68315:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x206:0x0] 00000008:00010000:1.0:1787826350.371060:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826350.371064:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248930, osc lock ffff907d962f4750, flags 0x400000400001002 00000008:00010000:1.0:1787826350.371071:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826350.371072:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248970, osc lock ffff907d962f55f0, flags 0x400000400001002 00010000:00010000:1.0:1787826350.371088:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826350.371101:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 67108863) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00200000:1.0:1787826350.371111:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826350.371115:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1787826350.371123:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826350.371139:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1787826350.371148:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (67108864 134217727) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00000020:00200000:1.0:1787826350.371156:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826350.371159:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1787826350.371172:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000020:00200000:1.0:1787826350.371181:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826350.371186:0:68315:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:1.0:1787826350.371190:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_release():245 00000020:00010000:1.0:1787826350.371193:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.371197:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248930 (ffff907da5608ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.371201:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248970 (ffff907da5608d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.371203:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.371206:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248930 (ffff907d884a7908/1) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.371209:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248970 (ffff907d884a7908/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826350.371217:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826350.371230:0:68315:0:(file.c:5224:ll_file_seek()) START file f204d.sanity-flr:[0x200000404:0x206:0x0], offset: 70254592, type: SEEK_DATA 00000080:00200000:1.0:1787826350.371235:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:1.0:1787826350.371239:0:68315:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826350.371241:0:68315:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826350.371245:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:1.0:1787826350.371249:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:1.0:1787826350.371261:0:68315:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 2048 00000080:00200000:1.0:1787826350.371269:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826350.371273:0:68315:0:(file.c:1848:ll_io_set_mirror()) f204d.sanity-flr: desiginated mirror: 1 00000080:00200000:1.0:1787826350.371275:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826350.371280:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:1.0:1787826350.371283:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:1.0:1787826350.371286:0:68315:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [17152, 18446744073709551615] 00000008:00010000:1.0:1787826350.371292:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:1.0:1787826350.371294:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c32630, osc lock ffff907d962f55f0, flags 0x0 00010000:00010000:1.0:1787826350.371302:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1787826350.371312:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (70254592 134217727) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1787826350.371321:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1787826350.371329:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1787826350.371336:0:68315:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00000080:00200000:1.0:1787826350.371345:0:68315:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:1.0:1787826350.371350:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b17b8 (ffff907d884a7908/0) at cl_lock_release():245 00000020:00010000:1.0:1787826350.371353:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b17b8 (ffff907d884a7908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.371356:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c32630 (ffff907da5608d88/1) at cl_lock_cancel():142 00010000:00010000:1.0:1787826350.371359:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1787826350.371367:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00000020:00010000:1.0:1787826350.371380:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b17b8 (ffff907d884a7908/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.371383:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c32630 (ffff907d884a7908/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826350.371389:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826350.371393:0:68315:0:(file.c:5262:ll_file_seek()) COMPLETED file f204d.sanity-flr:[0x200000404:0x206:0x0], offset: 70254592, type: SEEK_DATA, rc = -6 00000080:00200008:1.0:1787826350.371399:0:68315:0:(file.c:5224:ll_file_seek()) START file f204d.sanity-flr:[0x200000404:0x206:0x0], offset: 0, type: (null) 00000080:00200000:1.0:1787826350.371403:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:1.0:1787826350.371407:0:68315:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826350.371410:0:68315:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826350.371412:0:68315:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x206:0x0] 00000008:00010000:1.0:1787826350.371419:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826350.371421:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248930, osc lock ffff907d962f55f0, flags 0x400000400001002 00000008:00010000:1.0:1787826350.371424:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826350.371425:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248970, osc lock ffff907d962f4750, flags 0x400000400001002 00010000:00010000:1.0:1787826350.371432:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826350.371441:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 67108863) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000020:00200000:1.0:1787826350.371449:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826350.371453:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1787826350.371460:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1787826350.371478:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1787826350.371486:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (67108864 134217727) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00000020:00200000:1.0:1787826350.371494:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826350.371497:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 1 00010000:00010000:1.0:1787826350.371504:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00000020:00200000:1.0:1787826350.371513:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826350.371517:0:68315:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:1.0:1787826350.371520:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_release():245 00000020:00010000:1.0:1787826350.371523:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.371525:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248930 (ffff907da5608d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.372332:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248970 (ffff907da5608ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.372335:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.372337:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248930 (ffff907d884a7908/1) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.372341:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248970 (ffff907d884a7908/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826350.372349:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826350.372356:0:68315:0:(file.c:5262:ll_file_seek()) COMPLETED file f204d.sanity-flr:[0x200000404:0x206:0x0], offset: 0, type: (null), rc = 70254592 00000080:00200080:1.0:1787826350.372363:0:68315:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88) cmd=400866b1 arg=0 00000080:00200080:1.0:1787826350.396439:0:68315:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88) cmd=400866b1 arg=2 00000080:00200008:1.0:1787826350.396459:0:68315:0:(file.c:6313:ll_getattr_dentry()) START file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826350.396468:0:68315:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88),name=f204d.sanity-flr 00000002:00010000:1.0:1787826350.396489:0:68315:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x206:0x0]) in obj [0x200000404:0x206:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826350.396506:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826350.396520:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1787826350.397804:0:68315:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x206:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826350.397816:0:68315:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) 00000080:00010000:1.0:1787826350.397821:0:68315:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) for lock 0xe5f736940357ba10 00000080:00010000:1.0:1787826350.397829:0:68315:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba10 from it ffff9dd142b27cd8 00010000:00010000:1.0:1787826350.397833:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826350.397844:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1787826350.397871:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:1.0:1787826350.397883:0:68315:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826350.397886:0:68315:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826350.397890:0:68315:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x206:0x0] 00000008:00010000:1.0:1787826350.397910:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826350.397914:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248930, osc lock ffff907d962f4750, flags 0x400000400001002 00000008:00010000:1.0:1787826350.397921:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826350.397923:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d87248970, osc lock ffff907d962f55f0, flags 0x400000400001002 00010000:00010000:1.0:1787826350.397951:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1787826350.397965:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 67108863) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000020:00200000:1.0:1787826350.397977:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826350.397981:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1787826350.397989:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1787826350.398004:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00010000:00010000:1.0:1787826350.398014:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (67108864 134217727) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00000020:00200000:1.0:1787826350.398021:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1787826350.398024:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 1 00010000:00010000:1.0:1787826350.398032:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00000020:00200000:1.0:1787826350.398041:0:68315:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826350.398050:0:68315:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:1.0:1787826350.398054:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_release():245 00000020:00010000:1.0:1787826350.398057:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.398060:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248930 (ffff907da5608ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.398064:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d87248970 (ffff907da5608d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.398066:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.398069:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248930 (ffff907d884a7908/1) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.398073:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d87248970 (ffff907d884a7908/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826350.398082:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826350.398104:0:68315:0:(llite_lib.c:2427:ll_setattr_raw()) START file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88) current size 70254592, valid attrs 2068, mode 0, uid -1976721614, gid -1, new size 70254592, atime 1787826350.000000000, mtime 1787826350.000000000, ctime 1787826350.000000000 00010000:00010000:1.0:1787826350.398125:0:68315:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826350.398140:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826350.398183:0:68315:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1787826350.398190:0:68315:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826350.398193:0:68315:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a200/0xe5f736940357ba10 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc10470 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000020:00100000:1.0:1787826350.398212:0:68315:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1787826350.398224:0:68315:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a1a40 x1874670011452032/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826350.398247:0:68315:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011452032:192.168.203.104@tcp:36:lfs.0 00000100:00100000:2.0:1787826350.408159:0:68315:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d87248900 going to sleep for 13 seconds 00000100:00100000:3.0:1787826350.415381:0:68315:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011452032:192.168.203.104@tcp:36:lfs.0 00000080:00200000:3.0:1787826350.415438:0:68315:0:(file.c:1848:ll_io_set_mirror()) f204d.sanity-flr: desiginated mirror: 2 00000080:00200000:3.0:1787826350.415446:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 6 restore needed 0 00020000:00200000:3.0:1787826350.415475:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[4] flags 0x11 00000080:00200000:3.0:1787826350.415494:0:68315:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:3.0:1787826350.415527:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787826350.415533:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d866e9ab0, osc lock ffff907d963b9110, flags 0x0 00000008:00010000:3.0:1787826350.415538:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787826350.415540:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d866e9af0, osc lock ffff907d963b9860, flags 0x0 00010000:00010000:3.0:1787826350.415567:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826350.415588:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826350.415605:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826350.415921:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826350.415931:0:68315:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826350.416140:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1c60417/0xe5f736940357ba17 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826350.416152:0:68315:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1c60417/0xe5f736940357ba17 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [10485760->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826350.416171:0:68315:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1c60417/0xe5f736940357ba17 lrc: 3/0,1 mode: --/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [10485760->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826350.416199:0:68315:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011452160:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826350.416336:0:68315:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88d28f00 going to sleep for 11 seconds 00000100:00100000:2.0:1787826350.424708:0:68315:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011452160:192.168.203.104@tcp:101:lfs.0 00010000:00010000:2.0:1787826350.424813:0:68315:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1c60417/0xe5f736940357ba17 lrc: 4/0,1 mode: --/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1048c expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826350.424840:0:68315:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1c60417/0xe5f736940357ba17 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1048c expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826350.424855:0:68315:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1c60417/0xe5f736940357ba17 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1048c expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826350.424874:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1c60417/0xe5f736940357ba17 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1048c expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1787826350.424890:0:68315:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 70254592, setting rss=70254592; kms=70254592, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1c60417/0xe5f736940357ba17 lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc1048c expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826350.424906:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1c60417/0xe5f736940357ba17 lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1048c expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826350.424921:0:68315:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1c60417/0xe5f736940357ba17 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1048c expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826350.425076:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011452288:192.168.203.104@tcp:10:lfs.0 00000100:00100000:2.0:1787826350.434286:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011452288:192.168.203.104@tcp:10:lfs.0 00000080:00200000:0.0:1787826350.434408:0:68315:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x206:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:0.0:1787826350.434418:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_release():245 00000020:00010000:0.0:1787826350.434421:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1787826350.434427:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d866e9ab0 (ffff907d884a7368/1) at cl_lock_cancel():142 00010000:00010000:0.0:1787826350.434440:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826350.434458:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00010000:0.0:1787826350.434482:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d866e9af0 (ffff907d884a71b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1787826350.434488:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1c60417/0xe5f736940357ba17 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1048c expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.434498:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1c60417/0xe5f736940357ba17 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1048c expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:0.0:1787826350.434517:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1787826350.434521:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d866e9ab0 (ffff907d884a7d88/1) at cl_lock_fini():72 00000020:00010000:0.0:1787826350.434528:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d866e9af0 (ffff907d884a7d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1787826350.434551:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1787826350.434567:0:68315:0:(llite_lib.c:2686:ll_setattr_raw()) COMPLETED file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88) current size 70254592, valid attrs 10002168, mode 0, uid -1976721614, gid -1, new size 70254592, atime 1787826350.000000000, mtime 1787826350.000000000, ctime 1787826350.000000000, rc 0 00000080:00200080:0.0:1787826350.434595:0:68315:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88) cmd=400866b1 arg=0 00000080:00200008:0.0:1787826350.434659:0:68315:0:(llite_lib.c:2427:ll_setattr_raw()) START file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88) current size 70254592, valid attrs 101f0, mode 9dd1, uid -1827241336, gid -28547, new size 18446621467747638152, atime 1787826324.000000000, mtime 1787826337.000000000, ctime 1787826350.000000000 00000020:00100000:0.0:1787826350.434743:0:68315:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1787826350.434753:0:68315:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90847840 x1874670011452544/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787826350.434779:0:68315:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011452544:192.168.203.104@tcp:36:lfs.0 00000100:00100000:0.0:1787826350.434889:0:68315:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd300 going to sleep for 13 seconds 00000100:00100000:0.0:1787826350.437914:0:68315:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011452544:192.168.203.104@tcp:36:lfs.0 00000080:00200000:0.0:1787826350.437985:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:0.0:1787826350.438004:0:68315:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826350.438007:0:68315:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826350.438016:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826350.438032:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:0.0:1787826350.438042:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:0.0:1787826350.438047:0:68315:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:0.0:1787826350.438066:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1787826350.438072:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d998fd330, osc lock ffff907d96368618, flags 0x0 00000008:00010000:0.0:1787826350.438075:0:68315:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1787826350.438077:0:68315:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d998fd370, osc lock ffff907d96368888, flags 0x0 00010000:00010000:0.0:1787826350.438112:0:68315:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d91e6e400 type 11 mode 2 res 9663677440/807 (0 67108863) 00010000:00010000:0.0:1787826350.438168:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 3/0,1 mode: --/PW res: [0x240000400:0x327:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.438182:0:68315:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 3/0,1 mode: --/PW res: [0x240000400:0x327:0x0].0x0 rrc: 3 type: EXT [0->67108863] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.438194:0:68315:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 3/0,1 mode: --/PW res: [0x240000400:0x327:0x0].0x0 rrc: 3 type: EXT [0->67108863] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826350.438217:0:68315:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011452672:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826350.438303:0:68315:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd3c0 going to sleep for 11 seconds 00000100:00100000:2.0:1787826350.441654:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826350.441907:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025508352 00000100:00100000:3.0:1787826350.441921:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025508352 00000100:00100000:3.0:1787826350.442005:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7157 00000100:00100000:3.0:1787826350.442016:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91148780 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:9562:x1874670025508352:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826350.442041:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:3.0:1787826350.442306:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826350.442314:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91148780 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:9562:x1874670025508352:12345-192.168.203.104@tcp:104: Request processed in 300us (719us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826350.442323:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7157 00010000:00010000:0.0:1787826350.443788:0:60483:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826350.443802:0:60483:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1787826350.443817:0:60483:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0b000 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826350.443826:0:60483:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x327:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800428400000000 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00000080:00200000:0.0:1787826350.443859:0:60483:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1787826350.443878:0:60483:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1787826350.443962:0:60483:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 2/0,0 mode: --/PR res: [0x240000400:0x327:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1787826350.443972:0:60483:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826350.444015:0:60483:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x327:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1787826350.444025:0:60483:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000008408fc3d/0xe5f736940357b9f4 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x327:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc10446 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:0.0:1787826350.444051:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826350.444054:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826350.444056:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826350.444286:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011452800:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826350.450683:0:68315:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011452672:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826350.450722:0:68315:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 4/0,1 mode: --/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->67108863] (req 0->67108863) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.450745:0:68315:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 4/0,1 mode: --/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->67108863] (req 0->67108863) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.450755:0:68315:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 4/0,1 mode: --/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->67108863] (req 0->67108863) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826350.453954:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826350.454705:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025508480 00000100:00100000:3.0:1787826350.454721:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025508480 00000100:00100000:3.0:1787826350.454793:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7158 00000100:00100000:3.0:1787826350.454804:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91148b40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:28563:x1874670025508480:12345-192.168.203.104@tcp:105: 00010000:00010000:3.0:1787826350.454824:0:40339:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 5/0,1 mode: --/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->67108863] (req 0->67108863) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.454850:0:40339:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 5/0,1 mode: --/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->67108863] (req 0->67108863) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.454863:0:40339:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 5/0,1 mode: --/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.454882:0:40339:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10080000020000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.454894:0:40339:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10080000020000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.454907:0:40339:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x80000020000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.454942:0:40339:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff907d8883a200) 00000100:00100000:3.0:1787826350.455026:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91148b40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:28563:x1874670025508480:12345-192.168.203.104@tcp:105: Request processed in 224us (1095us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826350.455035:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7158 00000100:00100000:1.0:1787826350.457716:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011452800:192.168.203.104@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1787826350.459005:0:68315:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.459031:0:68315:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.459055:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826350.459071:0:68315:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10020000020000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.459087:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.459096:0:68315:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.459138:0:68315:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d925e9400 type 11 mode 2 res 10737419264/782 (67108864 134217727) 00010000:00010000:0.0:1787826350.459193:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.459203:0:68315:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [67108864->134217727] (req 67108864->134217727) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.459220:0:68315:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [67108864->134217727] (req 67108864->134217727) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826350.459251:0:68315:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011452928:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826350.459370:0:68315:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd3c0 going to sleep for 11 seconds 00000100:00100000:2.0:1787826350.461197:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826350.461492:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025508608 00000100:00100000:3.0:1787826350.461503:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025508608 00000100:00100000:3.0:1787826350.461576:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7159 00000100:00100000:3.0:1787826350.461587:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d9114a940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:36728:x1874670025508608:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826350.461652:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1787826350.463182:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826350.463200:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d9114a940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:36728:x1874670025508608:12345-192.168.203.104@tcp:104: Request processed in 1610us (2008us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826350.463212:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7159 00010000:00010000:0.0:1787826350.463660:0:40347:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826350.463671:0:40347:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00010000:00010000:0.0:1787826350.463686:0:40347:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0a000 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826350.463694:0:40347:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800428400000000 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00000080:00200000:0.0:1787826350.463719:0:40347:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1787826350.463747:0:40347:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1787826350.463811:0:40347:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 2/0,0 mode: --/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00010000:00010000:0.0:1787826350.463820:0:40347:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826350.463850:0:40347:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 1/0,0 mode: --/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00010000:00010000:0.0:1787826350.463861:0:40347:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000b738ad6f/0xe5f736940357b9fb lrc: 0/0,0 mode: --/PR res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc1044d expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 11 lru_type: 1 00010000:00010000:0.0:1787826350.463887:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826350.463889:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826350.463891:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826350.463930:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011453056:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826350.464511:0:68315:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011452928:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826350.464534:0:68315:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [67108864->134217727] (req 67108864->134217727) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.464546:0:68315:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [67108864->134217727] (req 67108864->134217727) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.464556:0:68315:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [67108864->134217727] (req 67108864->134217727) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826350.467022:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826350.467857:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025508736 00000100:00100000:3.0:1787826350.467871:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025508736 00000100:00100000:3.0:1787826350.467959:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7160 00000100:00100000:3.0:1787826350.467973:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91148f00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:5934:x1874670025508736:12345-192.168.203.104@tcp:105: 00010000:00010000:3.0:1787826350.467997:0:40339:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [67108864->134217727] (req 67108864->134217727) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.468012:0:40339:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [67108864->134217727] (req 67108864->134217727) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.468025:0:40339:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 5/0,1 mode: --/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.468040:0:40339:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x10080000020000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.468053:0:40339:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x10080000020000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.468065:0:40339:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x80000020000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.468425:0:40339:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff907d8883a600) 00000100:00100000:3.0:1787826350.468511:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91148f00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:5934:x1874670025508736:12345-192.168.203.104@tcp:105: Request processed in 539us (1500us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826350.468521:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7160 00000100:00100000:1.0:1787826350.476782:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011453056:192.168.203.104@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1787826350.481659:0:68315:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.481690:0:68315:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.481712:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826350.481738:0:68315:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 70254592, setting rss=70254592; kms=70254592, end=18446744073709551615 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x10020000020000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.481754:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.481764:0:68315:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826350.481974:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011453184:192.168.203.104@tcp:2:lfs.0 00000100:00100000:1.0:1787826350.498069:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011453312:192.168.203.104@tcp:2:lfs.0 00000100:00100000:1.0:1787826350.505420:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011453312:192.168.203.104@tcp:2:lfs.0 00000100:00100000:0.0:1787826350.506364:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011453184:192.168.203.104@tcp:2:lfs.0 00000020:00010000:1.0:1787826350.506525:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_release():245 00000020:00010000:1.0:1787826350.506530:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826350.506535:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d998fd330 (ffff907d884a7368/1) at cl_lock_cancel():142 00010000:00010000:1.0:1787826350.506546:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.506562:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:1.0:1787826350.506579:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d998fd370 (ffff907d884a71b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1787826350.506584:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.506594:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:1.0:1787826350.506603:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.506621:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d998fd330 (ffff907d884a7d88/1) at cl_lock_fini():72 00000020:00010000:1.0:1787826350.506627:0:68315:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d998fd370 (ffff907d884a7d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826350.506646:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826350.506657:0:68315:0:(llite_lib.c:2686:ll_setattr_raw()) COMPLETED file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88) current size 70254592, valid attrs 100101f0, mode 9dd1, uid -1827241336, gid -28547, new size 18446621467747638152, atime 1787826324.000000000, mtime 1787826337.000000000, ctime 1787826350.000000000, rc 0 00000080:00200080:1.0:1787826350.506677:0:68315:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88) cmd=c00c66f3 arg=20d82a0 00010000:00010000:1.0:1787826350.506902:0:68315:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b9df lrc: 2/0,0 mode: PW/PW res: [0x200000404:0x206:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x81008000000000 nid: local remote: 0x7918213f4fc1040e expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.506920:0:68315:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357b9df lrc: 0/0,0 mode: --/PW res: [0x200000404:0x206:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x85809400000000 nid: local remote: 0x7918213f4fc1040e expref: -99 pid: 68315 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826350.506951:0:68315:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d908a1e00 x1874670011448960/t8589943008(8589943008) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826367 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826350.506962:0:68315:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826350.506968:0:68315:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a2d00 x1874670011453568/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 504/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826350.506991:0:68315:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011453568:192.168.203.104@tcp:35:lfs.0 00000100:00100000:1.0:1787826350.507142:0:68315:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d872489c0 going to sleep for 11 seconds 00000100:00100000:2.0:1787826350.509145:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826350.509475:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025508864 00000100:00100000:3.0:1787826350.509489:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025508864 00000100:00100000:3.0:1787826350.509549:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7161 00000100:00100000:3.0:1787826350.509560:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d9114a1c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32459:x1874670025508864:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826350.509577:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1787826350.509807:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826350.509814:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d9114a1c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32459:x1874670025508864:12345-192.168.203.104@tcp:104: Request processed in 257us (681us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826350.509821:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7161 00010000:00010000:1.0:1787826350.509965:0:61297:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826350.509971:0:61297:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826350.509981:0:61297:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0ae00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826350.509985:0:61297:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826350.509992:0:61297:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826350.510000:0:61297:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 3/0,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1787826350.510020:0:61297:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x206:0x0]: losing layout lock 00010000:00010000:1.0:1787826350.510055:0:61297:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 2/0,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826350.510062:0:61297:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1787826350.510113:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011453696:192.168.203.104@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1787826350.510286:0:61297:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 1/0,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826350.510294:0:61297:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357b9e6 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc10431 expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1787826350.510308:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826350.510310:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826350.510312:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826350.513993:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011453696:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826350.522352:0:68315:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589943015 gen 1 00000100:00100000:1.0:1787826350.522397:0:68315:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011453568:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826350.522432:0:68315:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d908a1e00 x1874670011448960/t8589943008(8589943008) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826367 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:1.0:1787826350.522473:0:68315:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 31 res 8589935620/518 (0 0) 00000080:00010000:1.0:1787826350.522486:0:68315:0:(file.c:7076:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000404:0x206:0x0](ffff907d93315b88) 00000002:00010000:1.0:1787826350.522499:0:68315:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x206:0x0]) in obj [0x200000404:0x206:0x0], intent: layout flags 0 00010000:00010000:1.0:1787826350.522553:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.522568:0:68315:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.522580:0:68315:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68315 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826350.522600:0:68315:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011453824:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826350.523304:0:68315:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d872489c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826350.527586:0:68315:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011453824:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826350.527667:0:68315:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc104af expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.527687:0:68315:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc104af expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826350.527713:0:68315:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x400 disposition=0x1, status=0 req@ffff907d908a2d00 x1874670011453824/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/896 e 0 to 0 dl 1787826368 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826350.527732:0:68315:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc104af expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826350.527748:0:68315:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc104af expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.527763:0:68315:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/518 (0 0) 00000080:00200000:1.0:1787826350.527770:0:68315:0:(namei.c:952:ll_intent_lock()) intent lock 1024 on i1 [0x200000404:0x206:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1787826350.527790:0:68315:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) for lock 0xe5f736940357ba2c 00000080:00010000:1.0:1787826350.527800:0:68315:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba2c from it ffff9dd142b27bc8 00010000:00010000:1.0:1787826350.527804:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc104af expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.527815:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc104af expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826350.527841:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc104af expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826350.527853:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc104af expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:1.0:1787826350.527862:0:68315:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x206:0x0](00000000407d6ff3) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc104af expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826350.527910:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc104af expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826350.527918:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc104af expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826350.527945:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826350.527957:0:68315:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826350.527961:0:68315:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826350.527969:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1787826350.527980:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:1.0:1787826350.527986:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:1.0:1787826350.528013:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1787826350.528025:0:68315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc104af expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826350.528035:0:68315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc104af expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:1.0:1787826350.528043:0:68315:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x206:0x0](00000000407d6ff3) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc104af expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826350.528059:0:68315:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 7 restore needed 0 00020000:00200000:1.0:1787826350.528064:0:68315:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826350.528067:0:68315:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826350.528069:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1787826350.528072:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:1.0:1787826350.528076:0:68315:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:1.0:1787826350.528082:0:68315:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1787826350.528097:0:68315:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 0 blocks 0 [cma]time 1787826350 1787826337 1787826324 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x327:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:1.0:1787826350.528129:0:68315:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 70254592 blocks 2048 [cma]time: 1787826337 1787826337 0 by oinfo size 70254592 blocks 2048 [cma]time 1787826350 1787826337 1787826324 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:1.0:1787826350.528147:0:68315:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 0 blocks 0 [cma]time 0 0 0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:1.0:1787826350.528163:0:68315:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 70254592 blocks 2048 [cma]time: 1787826350 1787826350 0 by oinfo size 70254592 blocks 2048 [cma]time 1787826350 1787826350 0 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1c60417/0xe5f736940357ba17 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1048c expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826350.528223:0:68315:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x206:0x0]: layout version change: 4294967294 -> 8 00010000:00010000:1.0:1787826350.528228:0:68315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc104af expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826350.528236:0:68315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc104af expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:1.0:1787826350.528279:0:68315:0:(file.c:479:ll_file_release()) START file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88), flags 154002 00000002:00100000:1.0:1787826350.528327:0:68315:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d83524780 x1874670011448576/t8589943007(8589943007) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826367 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826350.528345:0:68315:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826350.528351:0:68315:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a2d00 x1874670011454080/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826350.528376:0:68315:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011454080:192.168.203.104@tcp:35:lfs.0 00000100:00100000:1.0:1787826350.528476:0:68315:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d872489c0 going to sleep for 11 seconds 00000100:00100000:1.0:1787826350.544442:0:68315:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68315:1874670011454080:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826350.544485:0:68315:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d83524780 x1874670011448576/t8589943007(8589943007) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826367 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826350.579769:0:68316:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826350.579805:0:68316:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826350.579828:0:68316:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:0.0:1787826350.579846:0:68316:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:0.0:1787826350.579855:0:68316:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826350.579864:0:68316:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826350.579868:0:68316:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:0.0:1787826350.579876:0:68316:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142bb7bf8 00010000:00010000:0.0:1787826350.579881:0:68316:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:0.0:1787826350.579892:0:68316:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:0.0:1787826350.579911:0:68316:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826350.579920:0:68316:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:0.0:1787826350.579925:0:68316:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:0.0:1787826350.579949:0:68316:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), file ffff907d90bfd500, open_flags 18800, mode 0 opened 0 00000080:00200000:0.0:1787826350.579957:0:68316:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), intent=open 00000002:00010000:0.0:1787826350.579973:0:68316:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x205:0x0], intent: open flags 0304001 00010000:00010000:0.0:1787826350.580036:0:68316:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba33 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 68316 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.580044:0:68316:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba33 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68316 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826350.580059:0:68316:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826350.580073:0:68316:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908470c0 x1874670011454336/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826350.580084:0:68316:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba33 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68316 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826350.580110:0:68316:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68316:1874670011454336:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826350.580346:0:68316:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd3c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826350.587030:0:68316:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68316:1874670011454336:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826350.587059:0:68316:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba33 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68316 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.587076:0:68316:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba33 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 68316 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.587084:0:68316:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba33 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 68316 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.587216:0:68316:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba33 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 68316 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.587228:0:68316:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba33 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 68316 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.587235:0:68316:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883aa00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826350.587240:0:68316:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba33 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 68316 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.587247:0:68316:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba33 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 68316 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.587255:0:68316:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba33 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 68316 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.587263:0:68316:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826350.587272:0:68316:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba33 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 68316 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.587280:0:68316:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba33 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 68316 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826350.587304:0:68316:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908470c0 x1874670011454336/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826368 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826350.587320:0:68316:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x205:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1787826350.587480:0:68316:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826350.587504:0:68316:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826350.587522:0:68316:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:0.0:1787826350.587535:0:68316:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:0.0:1787826350.587543:0:68316:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826350.587548:0:68316:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826350.587552:0:68316:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:0.0:1787826350.587558:0:68316:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142bb7bf8 00010000:00010000:0.0:1787826350.587562:0:68316:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:0.0:1787826350.587571:0:68316:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:0.0:1787826350.587586:0:68316:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826350.587594:0:68316:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=267 00000080:00200000:0.0:1787826350.587602:0:68316:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 44 00000080:00200008:0.0:1787826350.587646:0:68316:0:(file.c:1090:ll_file_open()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00000080:00200080:0.0:1787826350.589085:0:68316:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908) cmd=c0086915 arg=25743d0 00000100:00100000:0.0:1787826350.589233:0:68316:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68316:1874670011454464:192.168.203.104@tcp:34:lfs.0 00000100:00100000:0.0:1787826350.589411:0:68316:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fd3c0 going to sleep for 13 seconds 00000100:00100000:2.0:1787826350.602135:0:68316:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68316:1874670011454464:192.168.203.104@tcp:34:lfs.0 00000080:00200008:2.0:1787826350.602477:0:68316:0:(file.c:479:ll_file_release()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00010000:00010000:2.0:1787826350.602513:0:68316:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b9b5 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x205:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10358 expref: -99 pid: 68307 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200008:0.0:1787826350.643485:0:68318:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1787826350.643498:0:68318:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826350.643525:0:68318:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826350.643549:0:68318:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:0.0:1787826350.643566:0:68318:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:0.0:1787826350.644618:0:68318:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826350.644640:0:68318:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826350.644645:0:68318:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:0.0:1787826350.644655:0:68318:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142acfcb8 00010000:00010000:0.0:1787826350.644660:0:68318:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:0.0:1787826350.644672:0:68318:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00000080:00200000:0.0:1787826350.645000:0:68318:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826350.645016:0:68318:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826350.645029:0:68318:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00010000:00010000:0.0:1787826350.645041:0:68318:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00000080:00200000:0.0:1787826350.645047:0:68318:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826350.645052:0:68318:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826350.645055:0:68318:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:0.0:1787826350.645069:0:68318:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142acfbb0 00010000:00010000:0.0:1787826350.645073:0:68318:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00010000:00010000:0.0:1787826350.645080:0:68318:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000080:00200000:0.0:1787826350.645091:0:68318:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:0.0:1787826350.645101:0:68318:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:0.0:1787826350.645126:0:68318:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffca8200b6c 00000080:00200008:0.0:1787826350.645141:0:68318:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:1.0:1787826350.648131:0:68320:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826350.648174:0:68320:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826350.648199:0:68320:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:1.0:1787826350.648224:0:68320:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000080:00200000:1.0:1787826350.648233:0:68320:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826350.648244:0:68320:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826350.648248:0:68320:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:1.0:1787826350.648256:0:68320:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142b77bf8 00010000:00010000:1.0:1787826350.648260:0:68320:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:1.0:1787826350.648270:0:68320:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00000080:00200000:1.0:1787826350.648287:0:68320:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826350.648296:0:68320:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:1.0:1787826350.648303:0:68320:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:1.0:1787826350.648317:0:68320:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), file ffff907d8500b080, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1787826350.648326:0:68320:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), intent=open 00000002:00010000:1.0:1787826350.648343:0:68320:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x205:0x0], intent: open flags 0304001 00010000:00010000:1.0:1787826350.648411:0:68320:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357ba3a lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 68320 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.648420:0:68320:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357ba3a lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68320 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826350.648433:0:68320:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826350.648448:0:68320:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a2d00 x1874670011454720/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826350.648459:0:68320:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357ba3a lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68320 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826350.648483:0:68320:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68320:1874670011454720:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826350.648986:0:68320:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d872489c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826350.668380:0:68320:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68320:1874670011454720:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826350.668410:0:68320:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357ba3a lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68320 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.668426:0:68320:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357ba3a lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 68320 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.668434:0:68320:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357ba3a lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 68320 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.668442:0:68320:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357ba3a lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 68320 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.668452:0:68320:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357ba3a lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 68320 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.668459:0:68320:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0bc00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826350.668466:0:68320:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357ba3a lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 68320 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.668473:0:68320:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357ba3a lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 68320 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.668482:0:68320:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357ba3a lrc: 5/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 68320 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.668490:0:68320:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826350.668498:0:68320:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357ba3a lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 68320 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826350.668518:0:68320:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357ba3a lrc: 0/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 68320 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826350.668540:0:68320:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908a2d00 x1874670011454720/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826368 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826350.668557:0:68320:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x205:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826350.669156:0:68320:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826350.669179:0:68320:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826350.669198:0:68320:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00010000:00010000:1.0:1787826350.669212:0:68320:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00000080:00200000:1.0:1787826350.669220:0:68320:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826350.669225:0:68320:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826350.669229:0:68320:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:1.0:1787826350.669235:0:68320:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142b77bf8 00010000:00010000:1.0:1787826350.669239:0:68320:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00010000:00010000:1.0:1787826350.669248:0:68320:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00000080:00200000:1.0:1787826350.669271:0:68320:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826350.669279:0:68320:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=267 00000080:00200000:1.0:1787826350.669287:0:68320:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 44 00000080:00200008:1.0:1787826350.669296:0:68320:0:(file.c:1090:ll_file_open()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00000080:00200080:1.0:1787826350.669335:0:68320:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908) cmd=c1406916 arg=22d3a20 00000100:00100000:1.0:1787826350.669404:0:68320:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68320:1874670011454848:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826350.669634:0:68320:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d872489c0 going to sleep for 13 seconds 00000100:00100000:1.0:1787826350.680234:0:68320:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68320:1874670011454848:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826350.680443:0:68320:0:(file.c:479:ll_file_release()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00010000:00010000:1.0:1787826350.680481:0:68320:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b9b5 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x205:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10358 expref: -99 pid: 68307 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200008:1.0:1787826350.722711:0:68325:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1787826350.724905:0:68325:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826350.724953:0:68325:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826350.724977:0:68325:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:1.0:1787826350.724998:0:68325:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00000080:00200000:1.0:1787826350.725007:0:68325:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826350.725015:0:68325:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826350.725019:0:68325:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:1.0:1787826350.725028:0:68325:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142bc7cb8 00010000:00010000:1.0:1787826350.725033:0:68325:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:1.0:1787826350.725043:0:68325:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00000080:00200000:1.0:1787826350.725256:0:68325:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826350.725264:0:68325:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826350.725271:0:68325:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00010000:00010000:1.0:1787826350.725281:0:68325:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00000080:00200000:1.0:1787826350.725288:0:68325:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826350.725291:0:68325:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826350.725294:0:68325:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:1.0:1787826350.725299:0:68325:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142bc7bb0 00010000:00010000:1.0:1787826350.725302:0:68325:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00010000:00010000:1.0:1787826350.725310:0:68325:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00000080:00200000:1.0:1787826350.725319:0:68325:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:1.0:1787826350.725328:0:68325:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:1.0:1787826350.725349:0:68325:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffcd1e3c35c 00000080:00200008:1.0:1787826350.725364:0:68325:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:3.0:1787826350.728904:0:68329:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826350.728948:0:68329:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826350.728970:0:68329:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00010000:00010000:3.0:1787826350.728988:0:68329:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00000080:00200000:3.0:1787826350.728997:0:68329:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826350.729008:0:68329:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826350.729012:0:68329:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:3.0:1787826350.729020:0:68329:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142befbf8 00010000:00010000:3.0:1787826350.729024:0:68329:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00010000:00010000:3.0:1787826350.729034:0:68329:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00000080:00200000:3.0:1787826350.729052:0:68329:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826350.729059:0:68329:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:3.0:1787826350.729067:0:68329:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:3.0:1787826350.729079:0:68329:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), file ffff907d85566300, open_flags 18800, mode 0 opened 0 00000080:00200000:3.0:1787826350.729087:0:68329:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), intent=open 00000002:00010000:3.0:1787826350.729102:0:68329:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x205:0x0], intent: open flags 0304001 00010000:00010000:3.0:1787826350.729166:0:68329:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c000/0xe5f736940357ba41 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 68329 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.729174:0:68329:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c000/0xe5f736940357ba41 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68329 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826350.729187:0:68329:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826350.729202:0:68329:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83524f00 x1874670011455104/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826350.729212:0:68329:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c000/0xe5f736940357ba41 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68329 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826350.729239:0:68329:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68329:1874670011455104:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826350.732016:0:68329:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d866e9f00 going to sleep for 13 seconds 00000100:00100000:3.0:1787826350.737376:0:68329:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68329:1874670011455104:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826350.737404:0:68329:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c000/0xe5f736940357ba41 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68329 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.737420:0:68329:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c000/0xe5f736940357ba41 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 68329 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.737428:0:68329:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c000/0xe5f736940357ba41 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 68329 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.737435:0:68329:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c000/0xe5f736940357ba41 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 68329 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.737446:0:68329:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c000/0xe5f736940357ba41 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 68329 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.737452:0:68329:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8629c000 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826350.737458:0:68329:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c000/0xe5f736940357ba41 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 68329 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.737466:0:68329:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c000/0xe5f736940357ba41 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 68329 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.737474:0:68329:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c000/0xe5f736940357ba41 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 68329 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.737482:0:68329:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826350.737490:0:68329:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c000/0xe5f736940357ba41 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 68329 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.737496:0:68329:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c000/0xe5f736940357ba41 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 68329 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826350.737569:0:68329:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d83524f00 x1874670011455104/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826368 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826350.737586:0:68329:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x205:0x0] suppgids 0 -1: rc -20 00000080:00200000:3.0:1787826350.738142:0:68329:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826350.738166:0:68329:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826350.738184:0:68329:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00010000:00010000:3.0:1787826350.738199:0:68329:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00000080:00200000:3.0:1787826350.738208:0:68329:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826350.738214:0:68329:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826350.738218:0:68329:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:3.0:1787826350.738224:0:68329:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142befbf8 00010000:00010000:3.0:1787826350.738228:0:68329:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00010000:00010000:3.0:1787826350.738237:0:68329:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00000080:00200000:3.0:1787826350.738255:0:68329:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826350.738263:0:68329:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=267 00000080:00200000:3.0:1787826350.738270:0:68329:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 44 00000080:00200008:3.0:1787826350.738280:0:68329:0:(file.c:1090:ll_file_open()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00000080:00200080:3.0:1787826350.738320:0:68329:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908) cmd=c1406916 arg=214da20 00000100:00100000:3.0:1787826350.738389:0:68329:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68329:1874670011455232:192.168.203.104@tcp:34:lfs.0 00000100:00100000:3.0:1787826350.738490:0:68329:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d866e9f00 going to sleep for 13 seconds 00000100:00100000:3.0:1787826350.747307:0:68329:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68329:1874670011455232:192.168.203.104@tcp:34:lfs.0 00000080:00200008:3.0:1787826350.747503:0:68329:0:(file.c:479:ll_file_release()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00010000:00010000:3.0:1787826350.747545:0:68329:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b9b5 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x205:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10358 expref: -99 pid: 68307 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200008:1.0:1787826350.800286:0:68332:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1787826350.800299:0:68332:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826350.800325:0:68332:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826350.800348:0:68332:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00010000:00010000:1.0:1787826350.800366:0:68332:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00000080:00200000:1.0:1787826350.800375:0:68332:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826350.800385:0:68332:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826350.800389:0:68332:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:1.0:1787826350.800398:0:68332:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142bf7cb8 00010000:00010000:1.0:1787826350.800402:0:68332:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00010000:00010000:1.0:1787826350.800412:0:68332:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00000080:00200000:1.0:1787826350.801902:0:68332:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826350.801925:0:68332:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826350.801950:0:68332:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00010000:00010000:1.0:1787826350.801964:0:68332:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00000080:00200000:1.0:1787826350.801972:0:68332:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826350.801979:0:68332:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826350.801982:0:68332:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:1.0:1787826350.801989:0:68332:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142bf7bb0 00010000:00010000:1.0:1787826350.801992:0:68332:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00010000:00010000:1.0:1787826350.802001:0:68332:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00000080:00200000:1.0:1787826350.802016:0:68332:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:1.0:1787826350.802026:0:68332:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:1.0:1787826350.802050:0:68332:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffc32d0d9dc 00000080:00200008:1.0:1787826350.802066:0:68332:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:0.0:1787826350.809705:0:68335:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826350.809750:0:68335:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826350.809774:0:68335:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00010000:00010000:0.0:1787826350.809793:0:68335:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00000080:00200000:0.0:1787826350.809802:0:68335:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826350.809813:0:68335:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826350.809817:0:68335:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:0.0:1787826350.809827:0:68335:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142c07bf8 00010000:00010000:0.0:1787826350.809831:0:68335:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00010000:00010000:0.0:1787826350.809841:0:68335:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00000080:00200000:0.0:1787826350.809862:0:68335:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826350.809870:0:68335:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:0.0:1787826350.809877:0:68335:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:0.0:1787826350.809889:0:68335:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), file ffff907d83cc3e00, open_flags 18800, mode 0 opened 0 00000080:00200000:0.0:1787826350.809898:0:68335:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), intent=open 00000002:00010000:0.0:1787826350.809912:0:68335:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x205:0x0], intent: open flags 0304001 00010000:00010000:0.0:1787826350.809990:0:68335:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba48 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 68335 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.809999:0:68335:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba48 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68335 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826350.810011:0:68335:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826350.810027:0:68335:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90847c00 x1874670011455488/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826350.810038:0:68335:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba48 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68335 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826350.810062:0:68335:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68335:1874670011455488:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826350.810309:0:68335:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fde40 going to sleep for 13 seconds 00000100:00100000:0.0:1787826350.821112:0:68335:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68335:1874670011455488:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826350.821140:0:68335:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba48 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68335 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.821156:0:68335:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba48 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 68335 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.821169:0:68335:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba48 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 68335 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.821176:0:68335:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba48 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 68335 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.821184:0:68335:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba48 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 68335 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.821191:0:68335:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883aa00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826350.821197:0:68335:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba48 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 68335 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.821205:0:68335:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba48 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 68335 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.821213:0:68335:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba48 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 68335 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.821219:0:68335:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826350.821227:0:68335:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba48 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 68335 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.821233:0:68335:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba48 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 68335 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826350.821256:0:68335:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d90847c00 x1874670011455488/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826368 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826350.821273:0:68335:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x205:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1787826350.821577:0:68335:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826350.821600:0:68335:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826350.821662:0:68335:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00010000:00010000:0.0:1787826350.821676:0:68335:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00000080:00200000:0.0:1787826350.821684:0:68335:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826350.821690:0:68335:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826350.821694:0:68335:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:0.0:1787826350.821701:0:68335:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142c07bf8 00010000:00010000:0.0:1787826350.821705:0:68335:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00010000:00010000:0.0:1787826350.821714:0:68335:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00000080:00200000:0.0:1787826350.821740:0:68335:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826350.821749:0:68335:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=267 00000080:00200000:0.0:1787826350.821756:0:68335:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 44 00000080:00200008:0.0:1787826350.821767:0:68335:0:(file.c:1090:ll_file_open()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00000080:00200080:0.0:1787826350.821807:0:68335:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908) cmd=c1406916 arg=1dc3a20 00000100:00100000:0.0:1787826350.822092:0:68335:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68335:1874670011455616:192.168.203.104@tcp:34:lfs.0 00000100:00100000:0.0:1787826350.822191:0:68335:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fde40 going to sleep for 13 seconds 00000100:00100000:0.0:1787826350.826039:0:68335:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68335:1874670011455616:192.168.203.104@tcp:34:lfs.0 00000080:00200008:0.0:1787826350.826123:0:68335:0:(file.c:479:ll_file_release()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00010000:00010000:0.0:1787826350.826159:0:68335:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b9b5 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x205:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10358 expref: -99 pid: 68307 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200008:2.0:1787826350.863319:0:68339:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1787826350.863333:0:68339:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826350.863358:0:68339:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826350.863381:0:68339:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00010000:00010000:2.0:1787826350.863406:0:68339:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00000080:00200000:2.0:1787826350.863414:0:68339:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826350.863422:0:68339:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826350.863426:0:68339:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:2.0:1787826350.863435:0:68339:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142be7cb8 00010000:00010000:2.0:1787826350.863439:0:68339:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00010000:00010000:2.0:1787826350.863449:0:68339:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00000080:00200000:2.0:1787826350.863598:0:68339:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826350.863686:0:68339:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826350.863694:0:68339:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00010000:00010000:2.0:1787826350.863704:0:68339:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00000080:00200000:2.0:1787826350.863711:0:68339:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826350.863716:0:68339:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826350.863718:0:68339:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:2.0:1787826350.865277:0:68339:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142be7bb0 00010000:00010000:2.0:1787826350.865283:0:68339:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00010000:00010000:2.0:1787826350.865297:0:68339:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00000080:00200000:2.0:1787826350.865314:0:68339:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:2.0:1787826350.865327:0:68339:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:2.0:1787826350.865426:0:68339:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffd755f476c 00000080:00200008:2.0:1787826350.865448:0:68339:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:0.0:1787826350.867602:0:68343:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826350.867649:0:68343:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826350.867669:0:68343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00010000:00010000:0.0:1787826350.867686:0:68343:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00000080:00200000:0.0:1787826350.867694:0:68343:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826350.867704:0:68343:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826350.867708:0:68343:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:0.0:1787826350.867716:0:68343:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142c1fbf8 00010000:00010000:0.0:1787826350.867733:0:68343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00010000:00010000:0.0:1787826350.867744:0:68343:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00000080:00200000:0.0:1787826350.867759:0:68343:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826350.867768:0:68343:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:0.0:1787826350.867774:0:68343:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:0.0:1787826350.867785:0:68343:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), file ffff907d855a8600, open_flags 18800, mode 0 opened 0 00000080:00200000:0.0:1787826350.867793:0:68343:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), intent=open 00000002:00010000:0.0:1787826350.867808:0:68343:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x205:0x0], intent: open flags 0304001 00010000:00010000:0.0:1787826350.867869:0:68343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba4f lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 68343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.867882:0:68343:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba4f lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68343 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826350.867895:0:68343:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826350.867908:0:68343:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90847c00 x1874670011455872/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826350.867919:0:68343:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba4f lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68343 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826350.867958:0:68343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68343:1874670011455872:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826350.868197:0:68343:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fde40 going to sleep for 13 seconds 00000100:00100000:0.0:1787826350.871949:0:68343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68343:1874670011455872:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826350.871981:0:68343:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba4f lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.871997:0:68343:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba4f lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 68343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.872004:0:68343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba4f lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 68343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.872013:0:68343:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba4f lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 68343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.872022:0:68343:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba4f lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 68343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.872028:0:68343:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883aa00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826350.872035:0:68343:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba4f lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 68343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.872042:0:68343:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba4f lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 68343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.872050:0:68343:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba4f lrc: 5/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 68343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.872058:0:68343:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826350.872065:0:68343:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba4f lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 68343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826350.872071:0:68343:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357ba4f lrc: 0/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 68343 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826350.872093:0:68343:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d90847c00 x1874670011455872/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826368 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826350.872109:0:68343:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x205:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1787826350.872391:0:68343:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826350.872414:0:68343:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826350.872431:0:68343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00010000:00010000:0.0:1787826350.872445:0:68343:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00000080:00200000:0.0:1787826350.872452:0:68343:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826350.872458:0:68343:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826350.872462:0:68343:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:0.0:1787826350.872469:0:68343:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142c1fbf8 00010000:00010000:0.0:1787826350.872472:0:68343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00010000:00010000:0.0:1787826350.872481:0:68343:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 43 lru_type: 1 00000080:00200000:0.0:1787826350.872497:0:68343:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826350.872506:0:68343:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=267 00000080:00200000:0.0:1787826350.872513:0:68343:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 44 00000080:00200008:0.0:1787826350.872523:0:68343:0:(file.c:1090:ll_file_open()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00000080:00200080:0.0:1787826350.872562:0:68343:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908) cmd=c1406916 arg=1267a20 00000100:00100000:0.0:1787826350.872657:0:68343:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68343:1874670011456000:192.168.203.104@tcp:34:lfs.0 00000100:00100000:0.0:1787826350.878986:0:68343:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d998fde40 going to sleep for 13 seconds 00000100:00100000:0.0:1787826350.889090:0:68343:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68343:1874670011456000:192.168.203.104@tcp:34:lfs.0 00000080:00200008:0.0:1787826350.889283:0:68343:0:(file.c:479:ll_file_release()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00010000:00010000:0.0:1787826350.889320:0:68343:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b9b5 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x205:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10358 expref: -99 pid: 68307 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:3.0:1787826350.911150:0:68345:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826350.911184:0:68345:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826350.911203:0:68345:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 43 lru_type: 1 00010000:00010000:3.0:1787826350.911218:0:68345:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 43 lru_type: 1 00000080:00200000:3.0:1787826350.911224:0:68345:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826350.911231:0:68345:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826350.911235:0:68345:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:3.0:1787826350.911242:0:68345:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142be7bf8 00010000:00010000:3.0:1787826350.911245:0:68345:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 43 lru_type: 1 00010000:00010000:3.0:1787826350.911252:0:68345:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 44 lru_type: 1 00000080:00200000:3.0:1787826350.911266:0:68345:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826350.911273:0:68345:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:3.0:1787826350.911278:0:68345:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:3.0:1787826350.911287:0:68345:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), file ffff907d85152a80, open_flags 18800, mode 0 opened 0 00000080:00200000:3.0:1787826350.911293:0:68345:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), intent=open 00000002:00010000:3.0:1787826350.911306:0:68345:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x205:0x0], intent: open flags 0304001 00010000:00010000:3.0:1787826350.911359:0:68345:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c800/0xe5f736940357ba56 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 68345 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826350.911365:0:68345:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c800/0xe5f736940357ba56 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68345 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826350.911375:0:68345:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826350.911387:0:68345:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83524f00 x1874670011456256/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826350.911395:0:68345:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c800/0xe5f736940357ba56 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68345 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826350.911415:0:68345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68345:1874670011456256:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826350.912056:0:68345:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d866e9f00 going to sleep for 13 seconds 00000100:00100000:2.0:1787826350.915424:0:68345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68345:1874670011456256:192.168.203.104@tcp:101:lfs.0 00010000:00010000:2.0:1787826350.915452:0:68345:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c800/0xe5f736940357ba56 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68345 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826350.915467:0:68345:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c800/0xe5f736940357ba56 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 68345 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826350.915475:0:68345:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c800/0xe5f736940357ba56 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 68345 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826350.915483:0:68345:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c800/0xe5f736940357ba56 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 68345 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826350.915493:0:68345:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c800/0xe5f736940357ba56 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 68345 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826350.915500:0:68345:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8629c800 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:2.0:1787826350.915516:0:68345:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c800/0xe5f736940357ba56 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 68345 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826350.915524:0:68345:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c800/0xe5f736940357ba56 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 68345 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826350.915534:0:68345:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c800/0xe5f736940357ba56 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 68345 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826350.915542:0:68345:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1787826350.915550:0:68345:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c800/0xe5f736940357ba56 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 68345 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826350.915557:0:68345:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629c800/0xe5f736940357ba56 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 68345 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826350.915645:0:68345:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d83524f00 x1874670011456256/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826368 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1787826350.915663:0:68345:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x205:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1787826350.915836:0:68345:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826350.915861:0:68345:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826350.915880:0:68345:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 44 lru_type: 1 00010000:00010000:2.0:1787826350.915894:0:68345:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 44 lru_type: 1 00000080:00200000:2.0:1787826350.915903:0:68345:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826350.915909:0:68345:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826350.915912:0:68345:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:2.0:1787826350.915917:0:68345:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142be7bf8 00010000:00010000:2.0:1787826350.915920:0:68345:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 44 lru_type: 1 00010000:00010000:2.0:1787826350.915944:0:68345:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 45 lru_type: 1 00000080:00200000:2.0:1787826350.915967:0:68345:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826350.915978:0:68345:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=267 00000080:00200000:2.0:1787826350.915986:0:68345:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 44 00000080:00200008:2.0:1787826350.915997:0:68345:0:(file.c:1090:ll_file_open()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00000080:00200080:2.0:1787826350.916042:0:68345:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908) cmd=c0086915 arg=9dd3d0 00000100:00100000:2.0:1787826350.916116:0:68345:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68345:1874670011456384:192.168.203.104@tcp:34:lfs.0 00000100:00100000:2.0:1787826350.916224:0:68345:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88d28cc0 going to sleep for 13 seconds 00000100:00100000:2.0:1787826350.919160:0:68345:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:68345:1874670011456384:192.168.203.104@tcp:34:lfs.0 00000080:00200008:3.0:1787826350.919772:0:68345:0:(file.c:479:ll_file_release()) START file d204d.sanity-flr:[0x200000404:0x205:0x0](ffff907d93310908), flags 304000 00010000:00010000:3.0:1787826350.919809:0:68345:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0be00/0xe5f736940357b9b5 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x205:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10358 expref: -99 pid: 68307 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1787826350.968570:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826351.991774:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826353.042730:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826353.911203:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011456512:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826353.911300:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011456768:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826353.911336:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1787826353.911342:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011456640:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826353.911563:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011456896:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826353.922112:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011456640:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826353.922132:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011456896:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826353.922141:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011456512:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826353.922151:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011456768:192.168.203.104@tcp:400:kworker.0 00010000:00010000:0.0:1787826354.037680:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826355.068291:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826356.084735:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826357.109726:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1787826357.405920:0:68764:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826357.405949:0:68764:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826357.405967:0:68764:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 45 lru_type: 1 00010000:00010000:2.0:1787826357.405983:0:68764:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 45 lru_type: 1 00000080:00200000:2.0:1787826357.405991:0:68764:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826357.406000:0:68764:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826357.406004:0:68764:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:2.0:1787826357.406012:0:68764:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd141e27b58 00010000:00010000:2.0:1787826357.406016:0:68764:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 45 lru_type: 1 00010000:00010000:2.0:1787826357.406025:0:68764:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 46 lru_type: 1 00000080:00200000:2.0:1787826357.406043:0:68764:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826357.406050:0:68764:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=24 00000080:00200000:2.0:1787826357.406055:0:68764:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:2.0:1787826357.406064:0:68764:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), flags=8 00000080:00200000:2.0:1787826357.406068:0:68764:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908), intent=getattr 00000002:00010000:2.0:1787826357.406079:0:68764:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x205:0x0], intent: getattr flags 01 00010000:00010000:2.0:1787826357.406129:0:68764:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 68764 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826357.406136:0:68764:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68764 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826357.406144:0:68764:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 3/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68764 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826357.406169:0:68764:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:68764:1874670011457024:192.168.203.104@tcp:101:rm.0 00000100:00100000:2.0:1787826357.406272:0:68764:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d98a00d80 going to sleep for 13 seconds 00000100:00100000:3.0:1787826357.433505:0:68764:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589943016 gen 1 00000100:00100000:3.0:1787826357.433525:0:68764:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:68764:1874670011457024:192.168.203.104@tcp:101:rm.0 00010000:00010000:3.0:1787826357.433546:0:68764:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 4/1,0 mode: --/CR res: [0x200000404:0x205:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc105c7 expref: -99 pid: 68764 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826357.433563:0:68764:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 4/1,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc105c7 expref: -99 pid: 68764 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826357.433570:0:68764:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 4/1,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc105c7 expref: -99 pid: 68764 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826357.433581:0:68764:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc105c7 expref: -99 pid: 68764 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826357.433592:0:68764:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc105c7 expref: -99 pid: 68764 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826357.433635:0:68764:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc105c7 expref: -99 pid: 68764 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826357.433644:0:68764:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc105c7 expref: -99 pid: 68764 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826357.433652:0:68764:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc105c7 expref: -99 pid: 68764 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826357.433663:0:68764:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d91149680 x1874670011457024/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/1136 e 0 to 0 dl 1787826375 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1787826357.433673:0:68764:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc105c7 expref: -99 pid: 68764 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1787826357.433794:0:68764:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc105c7 expref: -99 pid: 68764 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826357.433806:0:68764:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/518 (0 0) 00000080:00200000:3.0:1787826357.433812:0:68764:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x205:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1787826357.433863:0:68764:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x206:0x0], updating i_size 70254592 00000080:00200000:3.0:1787826357.433867:0:68764:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x206:0x0](ffff907d93315b88): rc = 0 00000080:00200000:3.0:1787826357.433894:0:68764:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x206:0x0]: layout version change: 8 -> 8 00000080:00010000:3.0:1787826357.433901:0:68764:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) for lock 0xe5f736940357ba5d 00000080:00010000:3.0:1787826357.433913:0:68764:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) 00000080:00010000:3.0:1787826357.433917:0:68764:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba5d from it ffff9dd141e27b20 00010000:00010000:3.0:1787826357.433920:0:68764:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc105c7 expref: -99 pid: 68764 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826357.433930:0:68764:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc105c7 expref: -99 pid: 68764 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:3.0:1787826357.433967:0:68764:0:(file.c:6313:ll_getattr_dentry()) START file f204d.sanity-flr:[0x200000404:0x206:0x0](ffff907d93315b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787826357.433973:0:68764:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88),name=f204d.sanity-flr 00000002:00010000:3.0:1787826357.433991:0:68764:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x206:0x0]) in obj [0x200000404:0x205:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826357.434000:0:68764:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc105c7 expref: -99 pid: 68764 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826357.434010:0:68764:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc105c7 expref: -99 pid: 68764 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1787826357.434016:0:68764:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x205:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826357.434022:0:68764:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) 00000080:00010000:3.0:1787826357.434024:0:68764:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x206:0x0](ffff907d93315b88) for lock 0xe5f736940357ba5d 00000080:00010000:3.0:1787826357.434028:0:68764:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba5d from it ffff9dd141e27cb8 00010000:00010000:3.0:1787826357.434031:0:68764:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc105c7 expref: -99 pid: 68764 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826357.434038:0:68764:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc105c7 expref: -99 pid: 68764 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787826357.434097:0:68764:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826357.434105:0:68764:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826357.434112:0:68764:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 46 lru_type: 1 00010000:00010000:3.0:1787826357.434121:0:68764:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 46 lru_type: 1 00000080:00200000:3.0:1787826357.434126:0:68764:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826357.434130:0:68764:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826357.434132:0:68764:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:3.0:1787826357.434136:0:68764:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd141e27c10 00010000:00010000:3.0:1787826357.434138:0:68764:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 46 lru_type: 1 00010000:00010000:3.0:1787826357.434144:0:68764:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 47 lru_type: 1 00000080:00200000:3.0:1787826357.434151:0:68764:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826357.434157:0:68764:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=24 00000080:00200000:3.0:1787826357.434161:0:68764:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 1 00000080:00200000:3.0:1787826357.434167:0:68764:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204d.sanity-flr, flags=0 00000080:00200000:3.0:1787826357.434170:0:68764:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x205:0x0](ffff907d93310908), inode mode 41ed mask 3 00000080:00200000:3.0:1787826357.434174:0:68764:0:(namei.c:2418:ll_unlink()) VFS Op:name=f204d.sanity-flr, dir=[0x200000404:0x205:0x0](ffff907d93310908) 00010000:00010000:3.0:1787826357.434196:0:68764:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc104af expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1787826357.434209:0:68764:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x206:0x0]: losing layout lock 00010000:00010000:3.0:1787826357.434220:0:68764:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc105c7 expref: -99 pid: 68764 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787826357.435335:0:68764:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x206:0x0]: losing layout lock 00010000:00010000:3.0:1787826357.435349:0:68764:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b999 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x206:0x0].0x0 bits 0x20/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc102fd expref: -99 pid: 68279 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826357.435464:0:68764:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 1/0,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc104af expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826357.435472:0:68764:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 1/0,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc105c7 expref: -99 pid: 68764 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826357.435478:0:68764:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b999 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc102fd expref: -99 pid: 68279 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826357.435483:0:68764:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:3.0:1787826357.435486:0:68764:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357ba2c lrc: 0/0,0 mode: --/CR res: [0x200000404:0x206:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc104af expref: -99 pid: 68315 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826357.435499:0:68764:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357ba5d lrc: 0/0,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc105c7 expref: -99 pid: 68764 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826357.435508:0:68764:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ac00/0xe5f736940357b999 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x206:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc102fd expref: -99 pid: 68279 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000020:00100000:3.0:1787826357.435528:0:68764:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1787826357.435539:0:68764:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83527840 x1874670011457280/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 512/2200 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826357.435562:0:68764:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:68764:1874670011457280:192.168.203.104@tcp:36:rm.0 00000100:00100000:3.0:1787826357.435917:0:68764:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88ae7cc0 going to sleep for 13 seconds 00000100:00100000:3.0:1787826357.451401:0:68764:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:68764:1874670011457280:192.168.203.104@tcp:36:rm.0 00000080:00200000:3.0:1787826357.451467:0:68764:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1787826357.451480:0:68764:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x206:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:3.0:1787826357.451484:0:68764:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:3.0:1787826357.451491:0:68764:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1787826357.451501:0:68764:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:3.0:1787826357.451510:0:68764:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:3.0:1787826357.451530:0:68764:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x206:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787826357.451537:0:68764:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200000404:0x206:0x0](ffff907d93315b88) 00010000:00010000:1.0:1787826358.134642:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826359.030107:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011457408:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826359.030198:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011457664:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826359.030234:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1787826359.030241:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011457536:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826359.030268:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011457792:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826359.039109:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011457536:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826359.039125:0:2349:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0001-osc-ffff907d986d0000: committing for last_committed 17179873852 gen 1 00000100:00100000:1.0:1787826359.040018:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011457792:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826359.040041:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011457408:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826359.040052:0:2349:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff907d986d0000: committing for last_committed 21474840513 gen 3 00000100:00100000:1.0:1787826359.040064:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011457664:192.168.203.104@tcp:400:kworker.0 00000001:02000400:2.0:1787826359.150130:0:68903:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-flr test 204e: FLR write/stale/resync sel test with repeated comp ========================================================== 06:25:57 (1787826357) 00010000:00010000:3.0:1787826359.156948:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826360.181027:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826361.208076:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826362.105878:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826362.105996:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:1.0:1787826362.108309:0:40337:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025512320 00000100:00100000:1.0:1787826362.108332:0:40337:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025512320 00000100:00100000:1.0:1787826362.108411:0:40337:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 286 00000100:00100000:1.0:1787826362.108423:0:40337:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83525e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6935:x1874670025512320:12345-192.168.203.104@tcp:104: 00010000:00010000:1.0:1787826362.108448:0:40337:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826362.108553:0:40337:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1787826362.108561:0:40337:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83525e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6935:x1874670025512320:12345-192.168.203.104@tcp:104: Request processed in 141us (2564us total) trans 0 rc 0/0 00000100:00100000:1.0:1787826362.108568:0:40337:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 286 00010000:00010000:1.0:1787826362.108599:0:60835:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826362.108636:0:60835:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826362.108646:0:60835:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0a400 already unused, calling callback (ffffffffc118e410) 00010000:00010000:1.0:1787826362.108652:0:60835:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x428400010000 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826362.108706:0:60835:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826362.108715:0:60835:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826362.108737:0:60835:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826362.108746:0:60835:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000614550de/0xe5f736940357ba02 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x30d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc1045b expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826362.108770:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826362.108772:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826362.108774:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826362.108792:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011457920:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1787826362.111934:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:1.0:1787826362.112742:0:62746:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025512448 00000100:00100000:1.0:1787826362.112755:0:62746:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025512448 00000100:00100000:1.0:1787826362.112820:0:62746:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 287 00000100:00100000:1.0:1787826362.112831:0:62746:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83527c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:28560:x1874670025512448:12345-192.168.203.104@tcp:104: 00010000:00010000:1.0:1787826362.112866:0:62746:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x420000030000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826362.112962:0:62746:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1787826362.112969:0:62746:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83527c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:28560:x1874670025512448:12345-192.168.203.104@tcp:104: Request processed in 140us (1051us total) trans 0 rc 0/0 00000100:00100000:1.0:1787826362.112978:0:62746:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 287 00010000:00010000:0.0:1787826362.113158:0:60472:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826362.113168:0:60472:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x420000030000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826362.113186:0:60472:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a600 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826362.113193:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x428400030000 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826362.113251:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826362.113259:0:60472:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826362.113295:0:60472:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826362.113304:0:60472:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357ba25 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x30e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc104a1 expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826362.113341:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826362.113343:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826362.113345:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826362.113365:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011458048:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1787826362.119485:0:40338:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:2.0:1787826362.119522:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025511936 00000100:00100000:2.0:1787826362.119534:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025511936 00000100:00100000:2.0:1787826362.119643:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7162 00000100:00100000:2.0:1787826362.119656:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d88636450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:27403:x1874670025511936:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826362.119682:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x420000030000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826362.119799:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826362.119807:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d88636450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:27403:x1874670025511936:12345-192.168.203.104@tcp:104: Request processed in 155us (13956us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826362.119815:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7162 00010000:00010000:0.0:1787826362.120029:0:60483:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826362.120039:0:60483:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x420000030000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826362.120049:0:60483:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a200 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826362.120056:0:60483:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x428400030000 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826362.120120:0:60483:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 2/0,0 mode: --/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826362.120128:0:60483:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826362.120148:0:60483:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 1/0,0 mode: --/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826362.120155:0:60483:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000028225a89/0xe5f736940357ba1e lrc: 0/0,0 mode: --/PW res: [0x240000400:0x327:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc1049a expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826362.120182:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826362.120184:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826362.120185:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826362.120206:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011458176:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826362.120445:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011458048:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826362.125473:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011457920:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826362.132188:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011458176:192.168.203.104@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1787826362.228793:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826362.410990:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826362.412165:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025512704 00000100:00100000:2.0:1787826362.412175:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025512704 00000100:00100000:2.0:1787826362.412234:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7163 00000100:00100000:2.0:1787826362.412244:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83526940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:35506:x1874670025512704:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826362.412262:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1c60417/0xe5f736940357ba17 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc1048c expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826362.412340:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826362.412347:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83526940 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:35506:x1874670025512704:12345-192.168.203.104@tcp:104: Request processed in 105us (1366us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826362.412355:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7163 00010000:00010000:1.0:1787826362.412436:0:61297:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826362.412443:0:61297:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1c60417/0xe5f736940357ba17 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc1048c expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826362.412452:0:61297:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d863c1000 already unused, calling callback (ffffffffc118e410) 00010000:00010000:1.0:1787826362.412458:0:61297:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1c60417/0xe5f736940357ba17 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x428400010000 nid: local remote: 0x7918213f4fc1048c expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826362.412501:0:61297:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1c60417/0xe5f736940357ba17 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc1048c expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826362.412509:0:61297:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826362.412537:0:61297:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1c60417/0xe5f736940357ba17 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc1048c expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826362.412546:0:61297:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1c60417/0xe5f736940357ba17 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x328:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc1048c expref: -99 pid: 68315 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826362.412568:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826362.412570:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826362.412571:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826362.412800:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1787826362.412827:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011458304:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826362.419808:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011458304:192.168.203.104@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1787826363.253855:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826364.151230:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011458560:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826364.151328:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011458816:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826364.151701:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1787826364.151710:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011458432:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826364.151745:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011458688:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826364.156944:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011458688:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826364.156963:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011458816:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826364.166061:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011458432:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826364.166082:0:2350:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589943017 gen 1 00000100:00100000:3.0:1787826364.166100:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011458560:192.168.203.104@tcp:400:kworker.0 00010000:00010000:1.0:1787826364.276970:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826365.300956:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826366.324786:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826367.352211:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826368.372766:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826369.268971:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011459072:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826369.268971:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011458944:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826369.269079:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011459200:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826369.269565:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011459328:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826369.282223:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011459200:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826369.285956:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011459072:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826369.285977:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011459328:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826369.289792:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011458944:192.168.203.104@tcp:400:kworker.0 00010000:00010000:0.0:1787826369.399753:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826370.421504:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826371.445813:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826372.470756:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826373.492863:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826374.389161:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835243c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011459584:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826374.389295:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011459840:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826374.389340:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1787826374.389347:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011459456:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826374.389373:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011459712:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826374.397292:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011459456:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826374.397322:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011459712:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826374.397333:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835243c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011459584:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826374.397344:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011459840:192.168.203.104@tcp:400:kworker.0 00010000:00010000:3.0:1787826374.529135:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826375.540971:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826376.564874:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826377.588860:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826378.617395:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826379.515250:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011459968:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826379.515362:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011460224:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826379.515396:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1787826379.515402:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011460096:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826379.515422:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011460352:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826379.522021:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011460352:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826379.522042:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011460224:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826379.524970:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011460096:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826379.524987:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011459968:192.168.203.104@tcp:400:kworker.0 00010000:00010000:3.0:1787826379.636923:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826380.663497:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1787826381.607297:0:69376:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826381.607324:0:69376:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826381.607343:0:69376:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 47 lru_type: 1 00010000:00010000:2.0:1787826381.607358:0:69376:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 47 lru_type: 1 00000080:00200000:2.0:1787826381.607366:0:69376:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826381.607373:0:69376:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826381.607377:0:69376:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:2.0:1787826381.607384:0:69376:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142b37be0 00010000:00010000:2.0:1787826381.607388:0:69376:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 47 lru_type: 1 00010000:00010000:2.0:1787826381.607396:0:69376:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 48 lru_type: 1 00000080:00200000:2.0:1787826381.607410:0:69376:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826381.607423:0:69376:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d204e.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), flags=1536 00000080:00200000:2.0:1787826381.607427:0:69376:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826381.607433:0:69376:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826381.607439:0:69376:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 48 lru_type: 1 00010000:00010000:2.0:1787826381.607447:0:69376:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 48 lru_type: 1 00000080:00200000:2.0:1787826381.607459:0:69376:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826381.607462:0:69376:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826381.607465:0:69376:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:2.0:1787826381.607468:0:69376:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142b37ce0 00010000:00010000:2.0:1787826381.607471:0:69376:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 48 lru_type: 1 00010000:00010000:2.0:1787826381.607478:0:69376:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 49 lru_type: 1 00000080:00200000:2.0:1787826381.607486:0:69376:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 3 00000080:00200000:2.0:1787826381.607491:0:69376:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826381.607497:0:69376:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826381.607648:0:69376:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 49 lru_type: 1 00010000:00010000:2.0:1787826381.607656:0:69376:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 49 lru_type: 1 00000080:00200000:2.0:1787826381.607663:0:69376:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826381.607668:0:69376:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826381.607671:0:69376:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357b93e 00000080:00010000:2.0:1787826381.607674:0:69376:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357b93e from it ffff9dd142b37dc8 00010000:00010000:2.0:1787826381.607677:0:69376:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 49 lru_type: 1 00010000:00010000:2.0:1787826381.607688:0:69376:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 50 lru_type: 1 00000080:00200000:2.0:1787826381.607696:0:69376:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 3 00000080:00200000:2.0:1787826381.607702:0:69376:0:(namei.c:2239:do_mkdir()) VFS Op:name=d204e.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148) 00010000:00010000:2.0:1787826381.607804:0:69376:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 50 lru_type: 1 00010000:00010000:2.0:1787826381.607855:0:69376:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 50 lru_type: 1 00010000:00010000:2.0:1787826381.607862:0:69376:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1787826381.607865:0:69376:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883ba00/0xe5f736940357b93e lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc101de expref: -99 pid: 68250 timeout: 0 lvb_type: 0 lru_score: 50 lru_type: 1 00000020:00100000:2.0:1787826381.607892:0:69376:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1787826381.607904:0:69376:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114a1c0 x1874670011460608/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 568/2152 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787826381.607928:0:69376:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job mkdir:6f891bcb-15cd-41e0-96e2-db843619c1be:69376:1874670011460608:192.168.203.104@tcp:36:mkdir.0 00000100:00100000:2.0:1787826381.608034:0:69376:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d87248480 going to sleep for 13 seconds 00000100:00100000:3.0:1787826381.618112:0:69376:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a1c0 pname:cluuid:pid:xid:nid:opc:job mkdir:6f891bcb-15cd-41e0-96e2-db843619c1be:69376:1874670011460608:192.168.203.104@tcp:36:mkdir.0 00000080:00200000:3.0:1787826381.618163:0:69376:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8) 00000080:00200000:3.0:1787826381.618170:0:69376:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x208:0x0], updating i_size 4096 00000080:00200000:3.0:1787826381.655242:0:69379:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826381.655272:0:69379:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826381.655332:0:69379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826381.655342:0:69379:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826381.655351:0:69379:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826381.655379:0:69379:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69379:1874670011460736:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826381.655471:0:69379:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da0397240 going to sleep for 13 seconds 00000100:00100000:0.0:1787826381.660501:0:69379:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69379:1874670011460736:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826381.660538:0:69379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.661667:0:69379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.661683:0:69379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.661695:0:69379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.661710:0:69379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.661727:0:69379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.661735:0:69379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826381.661749:0:69379:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d83525a40 x1874670011460736/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/688 e 0 to 0 dl 1787826399 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826381.661764:0:69379:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.661778:0:69379:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1787826381.661785:0:69379:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1787826381.661823:0:69379:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:0.0:1787826381.661830:0:69379:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:0.0:1787826381.661836:0:69379:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826381.661839:0:69379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142ad7b58 00010000:00010000:0.0:1787826381.661842:0:69379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.661850:0:69379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826381.661875:0:69379:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826381.661897:0:69379:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d204e.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), flags=29 00000080:00200000:0.0:1787826381.661901:0:69379:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d204e.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), intent=0 00000002:00010000:0.0:1787826381.661919:0:69379:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: d204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00010000:00010000:0.0:1787826381.661958:0:69379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357ba6b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.661966:0:69379:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357ba6b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.662042:0:69379:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357ba6b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826381.662063:0:69379:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69379:1874670011460864:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826381.662293:0:69379:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88ae7240 going to sleep for 13 seconds 00000100:00100000:0.0:1787826381.670527:0:69379:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69379:1874670011460864:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826381.670561:0:69379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357ba6b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10637 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.670582:0:69379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357ba6b lrc: 4/1,0 mode: --/PR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10637 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.670590:0:69379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357ba6b lrc: 4/1,0 mode: --/PR res: [0x200000404:0x208:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10637 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.670601:0:69379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357ba6b lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10637 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.670644:0:69379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357ba6b lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10637 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.670656:0:69379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357ba6b lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10637 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.670664:0:69379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357ba6b lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10637 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.670671:0:69379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357ba6b lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10637 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826381.670681:0:69379:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x10 disposition=0xb, status=0 req@ffff907d90844f00 x1874670011460864/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/584 e 0 to 0 dl 1787826399 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826381.670693:0:69379:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357ba6b lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10637 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.670703:0:69379:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/520 (0 0) 00000080:00200000:0.0:1787826381.670709:0:69379:0:(namei.c:952:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1787826381.670738:0:69379:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x208:0x0], updating i_size 4096 00000080:00200000:0.0:1787826381.670742:0:69379:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x208:0x0](ffff907d933242c8): rc = 0 00000080:00010000:0.0:1787826381.670748:0:69379:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x208:0x0](ffff907d933242c8) for lock 0xe5f736940357ba6b 00000080:00010000:0.0:1787826381.670760:0:69379:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x208:0x0](ffff907d933242c8) 00000080:00010000:0.0:1787826381.670764:0:69379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba6b from it ffff9dd142ad7a28 00010000:00010000:0.0:1787826381.670767:0:69379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357ba6b lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10637 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.670776:0:69379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357ba6b lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10637 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826381.670808:0:69379:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:0.0:1787826381.670825:0:69379:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), flags=13 00000080:00200000:0.0:1787826381.670829:0:69379:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=getattr 00000002:00010000:0.0:1787826381.670845:0:69379:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x208:0x0], intent: getattr flags 01 00010000:00010000:0.0:1787826381.670885:0:69379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba72 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.670891:0:69379:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba72 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.670900:0:69379:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba72 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826381.670919:0:69379:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69379:1874670011460992:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826381.671007:0:69379:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238a80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826381.675935:0:69379:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69379:1874670011460992:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826381.675956:0:69379:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba72 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.675969:0:69379:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba72 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.675989:0:69379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba72 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.675996:0:69379:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba72 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.676010:0:69379:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba72 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.676016:0:69379:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a600 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826381.676021:0:69379:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba72 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.676027:0:69379:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba72 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.676033:0:69379:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba72 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.676039:0:69379:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826381.676046:0:69379:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba72 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.676051:0:69379:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba72 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826381.676069:0:69379:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff907d90844f00 x1874670011460992/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/584 e 0 to 0 dl 1787826399 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826381.676084:0:69379:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x208:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1787826381.676105:0:69379:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 30 res 8589935620/520 (0 0) 00000080:00200000:0.0:1787826381.676194:0:69379:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826381.676209:0:69379:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826381.676218:0:69379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826381.676229:0:69379:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826381.676236:0:69379:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826381.676240:0:69379:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826381.676243:0:69379:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:0.0:1787826381.676248:0:69379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142ad7be8 00010000:00010000:0.0:1787826381.676252:0:69379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826381.676260:0:69379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826381.676273:0:69379:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826381.676280:0:69379:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=16408 00000080:00200000:0.0:1787826381.676284:0:69379:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:0.0:1787826381.676304:0:69379:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), flags=16392 00000080:00200000:0.0:1787826381.676308:0:69379:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=getattr 00000002:00010000:0.0:1787826381.676318:0:69379:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x208:0x0], intent: getattr flags 01 00010000:00010000:0.0:1787826381.676350:0:69379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba79 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.676357:0:69379:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba79 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.676366:0:69379:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba79 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826381.676383:0:69379:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69379:1874670011461120:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826381.676461:0:69379:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238a80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826381.683025:0:69379:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69379:1874670011461120:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826381.683048:0:69379:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba79 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.683062:0:69379:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba79 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.683069:0:69379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba79 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.683076:0:69379:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba79 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.683084:0:69379:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba79 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.683090:0:69379:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a200 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826381.683094:0:69379:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba79 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.683100:0:69379:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba79 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.683108:0:69379:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba79 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.683115:0:69379:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826381.683121:0:69379:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba79 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.683128:0:69379:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba79 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826381.683147:0:69379:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff907d90844f00 x1874670011461120/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/584 e 0 to 0 dl 1787826399 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826381.683160:0:69379:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x208:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1787826381.683182:0:69379:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 30 res 8589935620/520 (0 0) 00000080:00200000:0.0:1787826381.683262:0:69379:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826381.683277:0:69379:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826381.683288:0:69379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826381.683299:0:69379:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826381.683305:0:69379:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826381.683309:0:69379:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826381.683313:0:69379:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:0.0:1787826381.683318:0:69379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142ad7b58 00010000:00010000:0.0:1787826381.683321:0:69379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826381.683328:0:69379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1787826381.683341:0:69379:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826381.683348:0:69379:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=13 00000080:00200008:0.0:1787826381.683357:0:69379:0:(file.c:6313:ll_getattr_dentry()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1787826381.683361:0:69379:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8),name=d204e.sanity-flr 00000002:00010000:0.0:1787826381.683370:0:69379:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x208:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826381.683375:0:69379:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 30 res 8589935620/520 (0 0) 00010000:00010000:0.0:1787826381.683409:0:69379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.683416:0:69379:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.683425:0:69379:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826381.683443:0:69379:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69379:1874670011461248:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826381.683538:0:69379:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238a80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826381.690162:0:69379:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69379:1874670011461248:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826381.690187:0:69379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1066f expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.690204:0:69379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1066f expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.690212:0:69379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x208:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1066f expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.690226:0:69379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1066f expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.690237:0:69379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1066f expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.690248:0:69379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1066f expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.690255:0:69379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1066f expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.690262:0:69379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1066f expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826381.690274:0:69379:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d90844f00 x1874670011461248/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/584 e 0 to 0 dl 1787826399 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826381.690286:0:69379:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1066f expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.690296:0:69379:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/520 (0 0) 00000080:00200000:0.0:1787826381.690302:0:69379:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1787826381.690323:0:69379:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x208:0x0], updating i_size 4096 00000080:00010000:0.0:1787826381.690328:0:69379:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x208:0x0](ffff907d933242c8) for lock 0xe5f736940357ba80 00000080:00010000:0.0:1787826381.690347:0:69379:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x208:0x0](ffff907d933242c8) 00000080:00010000:0.0:1787826381.690351:0:69379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba80 from it ffff9dd142ad7cb8 00010000:00010000:0.0:1787826381.690355:0:69379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1066f expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.690364:0:69379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1066f expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:0.0:1787826381.690941:0:69379:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1787826381.690948:0:69379:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826381.690963:0:69379:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826381.691705:0:69379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826381.691737:0:69379:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1787826381.691746:0:69379:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826381.691754:0:69379:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826381.691757:0:69379:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:0.0:1787826381.691763:0:69379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142ad7cb8 00010000:00010000:0.0:1787826381.691767:0:69379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826381.691774:0:69379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1787826381.691928:0:69379:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826381.691938:0:69379:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826381.691945:0:69379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1787826381.691954:0:69379:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1787826381.691961:0:69379:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826381.691965:0:69379:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826381.691967:0:69379:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:0.0:1787826381.693632:0:69379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142ad7b58 00010000:00010000:0.0:1787826381.693642:0:69379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1787826381.693653:0:69379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:0.0:1787826381.693669:0:69379:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826381.693680:0:69379:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=24 00000080:00200000:0.0:1787826381.693688:0:69379:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:0.0:1787826381.693704:0:69379:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), flags=8 00000080:00200000:0.0:1787826381.693709:0:69379:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=getattr 00000002:00010000:0.0:1787826381.693738:0:69379:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x208:0x0], intent: getattr flags 01 00010000:00010000:0.0:1787826381.693789:0:69379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba87 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.693798:0:69379:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba87 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.693808:0:69379:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba87 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826381.693849:0:69379:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69379:1874670011461376:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826381.694311:0:69379:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238a80 going to sleep for 13 seconds 00010000:00010000:1.0:1787826381.694444:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826381.705572:0:69379:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69379:1874670011461376:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826381.705594:0:69379:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba87 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.705642:0:69379:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba87 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.705649:0:69379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba87 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.705655:0:69379:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba87 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.705663:0:69379:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba87 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.705669:0:69379:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a600 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826381.705674:0:69379:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba87 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.705680:0:69379:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba87 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.705685:0:69379:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba87 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x20e400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.705691:0:69379:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826381.705698:0:69379:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba87 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.705704:0:69379:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357ba87 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x2 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826381.705731:0:69379:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffff907d90844f00 x1874670011461376/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/584 e 0 to 0 dl 1787826399 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826381.705746:0:69379:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x208:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1787826381.705782:0:69379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1066f expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826381.705796:0:69379:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1066f expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00010000:0.0:1787826381.705805:0:69379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba80 from it ffff9dd142ad7930 00010000:00010000:0.0:1787826381.705808:0:69379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1066f expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826381.705816:0:69379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1066f expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826381.705893:0:69379:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826381.705907:0:69379:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826381.705916:0:69379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1787826381.705924:0:69379:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:0.0:1787826381.705937:0:69379:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826381.705941:0:69379:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826381.705944:0:69379:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:0.0:1787826381.705949:0:69379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142ad7bf8 00010000:00010000:0.0:1787826381.705952:0:69379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1787826381.705959:0:69379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:0.0:1787826381.705968:0:69379:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826381.705976:0:69379:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=25 00000080:00200000:0.0:1787826381.705981:0:69379:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:0.0:1787826381.705987:0:69379:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204e.sanity-flr, flags=777 00000080:00200000:0.0:1787826381.705991:0:69379:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 3 00000080:00200000:0.0:1787826381.705995:0:69379:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), file ffff907d871ea600, open_flags a141, mode 81b6 opened 0 00000080:00200000:0.0:1787826381.706004:0:69379:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=open|creat 00000002:00010000:0.0:1787826381.706032:0:69379:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204e.sanity-flr,[0x200000404:0x209:0x0]) in obj [0x200000404:0x208:0x0], intent: open|creat flags 0120502 00010000:00010000:0.0:1787826381.706045:0:69379:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc1066f expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826381.706060:0:69379:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b200/0xe5f736940357ba6b lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x208:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10637 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826381.706106:0:69379:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x208:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc1066f expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826381.706117:0:69379:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826381.706120:0:69379:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba80 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x208:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc1066f expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826381.706143:0:69379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 3/0,1 mode: --/CW res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.706150:0:69379:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 3/0,1 mode: --/CW res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826381.706160:0:69379:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826381.706167:0:69379:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90844f00 x1874670011461632/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/67896 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826381.706176:0:69379:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 3/0,1 mode: --/CW res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826381.706197:0:69379:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69379:1874670011461632:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826381.706358:0:69379:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238a80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826381.713048:0:69379:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69379:1874670011461632:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826381.713080:0:69379:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 4/0,1 mode: --/CW res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc106a0 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.713102:0:69379:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 4/0,1 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc106a0 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.713110:0:69379:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 4/0,1 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc106a0 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.713124:0:69379:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc106a0 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.713135:0:69379:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc106a0 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.713148:0:69379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc106a0 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.713156:0:69379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc106a0 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.713163:0:69379:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc106a0 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826381.713174:0:69379:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x8000037, status=0 req@ffff907d90844f00 x1874670011461632/t8589943019(8589943019) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826399 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1787826381.713187:0:69379:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90844f00 x1874670011461632/t8589943019(8589943019) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826399 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826381.713199:0:69379:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc106a0 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.713210:0:69379:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/521 (0 0) 00000080:00200000:0.0:1787826381.713216:0:69379:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x200000404:0x208:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1787826381.713229:0:69379:0:(namei.c:1628:ll_create_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=open|creat 00000080:00200000:0.0:1787826381.713262:0:69379:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8) 00000080:00200000:0.0:1787826381.713268:0:69379:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x209:0x0], updating i_size 0 00000080:00200000:0.0:1787826381.713311:0:69379:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x209:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:0.0:1787826381.713317:0:69379:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) 00000080:00010000:0.0:1787826381.713320:0:69379:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) for lock 0xe5f736940357ba8e 00000080:00010000:0.0:1787826381.713330:0:69379:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) 00000080:00200008:0.0:1787826381.713342:0:69379:0:(file.c:1090:ll_file_open()) START file f204e.sanity-flr:[0x200000404:0x209:0x0](ffff907d933121c8), flags 120501 00000002:00100000:0.0:1787826381.713361:0:69379:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90844f00 x1874670011461632/t8589943019(8589943019) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826399 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:0.0:1787826381.713396:0:69379:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba8e from it ffff907d832412a0 00010000:00010000:0.0:1787826381.713401:0:69379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc106a0 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.713410:0:69379:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc106a0 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826381.713430:0:69379:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8), inode mode 81a4 mask 40 00000080:00200000:0.0:1787826381.713479:0:69379:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8), inode mode 81a4 mask 2 00000080:00200000:0.0:1787826381.713488:0:69379:0:(xattr.c:414:ll_xattr_set()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8), xattr lov 00000002:00010000:0.0:1787826381.713512:0:69379:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x209:0x0]) in obj [0x200000404:0x208:0x0], intent: open flags 040000002 00010000:00010000:0.0:1787826381.713558:0:69379:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba95 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.713566:0:69379:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba95 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826381.713575:0:69379:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826381.713582:0:69379:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90846940 x1874670011461888/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826381.713592:0:69379:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba95 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826381.713632:0:69379:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69379:1874670011461888:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826381.713843:0:69379:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238a80 going to sleep for 13 seconds 00000100:00100000:2.0:1787826381.741879:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826381.743905:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025516672 00000100:00100000:3.0:1787826381.743921:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025516672 00000100:00100000:3.0:1787826381.743994:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7164 00000100:00100000:3.0:1787826381.744004:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91148000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32860:x1874670025516672:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826381.744022:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc106a0 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826381.744117:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826381.744124:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91148000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:32860:x1874670025516672:12345-192.168.203.104@tcp:104: Request processed in 122us (2257us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826381.744131:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7164 00010000:00010000:1.0:1787826381.744542:0:60835:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826381.744552:0:60835:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc106a0 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826381.744573:0:60835:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a200 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826381.744578:0:60835:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc106a0 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826381.744586:0:60835:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc106a0 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826381.744600:0:60835:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 3/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x7918213f4fc106a0 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826381.744667:0:60835:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x209:0x0]: losing layout lock 00010000:00010000:1.0:1787826381.744710:0:60835:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 2/0,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc106a0 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826381.744728:0:60835:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826381.744749:0:60835:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 1/0,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc106a0 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826381.744768:0:60835:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a200/0xe5f736940357ba8e lrc: 0/0,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x1b rrc: 2 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc106a0 expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826381.744788:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826381.744790:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826381.744792:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826381.744800:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1787826381.744816:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011462016:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826381.750833:0:69379:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69379:1874670011461888:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826381.750858:0:69379:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba95 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.750872:0:69379:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba95 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.750879:0:69379:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba95 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.750885:0:69379:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba95 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.750892:0:69379:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba95 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.750898:0:69379:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883b600 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826381.750903:0:69379:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba95 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.750909:0:69379:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba95 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.750915:0:69379:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba95 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.750922:0:69379:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826381.750928:0:69379:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba95 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.750934:0:69379:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba95 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69379 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826381.750952:0:69379:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x800002b, status=0 req@ffff907d90846940 x1874670011461888/t8589943020(8589943020) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1056 e 0 to 0 dl 1787826399 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1787826381.750965:0:69379:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90846940 x1874670011461888/t8589943020(8589943020) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1056 e 0 to 0 dl 1787826399 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826381.750979:0:69379:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x208:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1787826381.751025:0:69379:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1787826381.751034:0:69379:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1787826381.751078:0:69379:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x209:0x0], updating i_size 0 00000002:00100000:0.0:1787826381.751095:0:69379:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90846940 x1874670011461888/t8589943020(8589943020) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 968/1056 e 0 to 0 dl 1787826399 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1787826381.751131:0:69379:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d90846940 x1874670011461888/t8589943020(8589943020) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 968/1056 e 0 to 0 dl 1787826399 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1787826381.751142:0:69379:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1787826381.751147:0:69379:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90847480 x1874670011462272/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1787826381.751166:0:69379:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69379:1874670011462272:192.168.203.104@tcp:35:lfs.0 00000100:00100000:0.0:1787826381.751255:0:69379:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238a80 going to sleep for 11 seconds 00000100:00100000:1.0:1787826381.751873:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011462016:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826381.756006:0:69379:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69379:1874670011462272:192.168.203.104@tcp:35:lfs.0 00000002:00100000:0.0:1787826381.756037:0:69379:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d90846940 x1874670011461888/t8589943020(8589943020) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 968/1056 e 0 to 0 dl 1787826399 ref 2 fl Complete:RQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:0.0:1787826381.756090:0:69379:0:(file.c:479:ll_file_release()) START file f204e.sanity-flr:[0x200000404:0x209:0x0](ffff907d933121c8), flags 100001 00000002:00100000:0.0:1787826381.756139:0:69379:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d90844f00 x1874670011461632/t8589943019(8589943019) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826399 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1787826381.756150:0:69379:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1787826381.756155:0:69379:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90846d00 x1874670011462528/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1787826381.756176:0:69379:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69379:1874670011462528:192.168.203.104@tcp:35:lfs.0 00000100:00100000:0.0:1787826381.756246:0:69379:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238a80 going to sleep for 11 seconds 00000100:00100000:0.0:1787826381.760129:0:69379:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69379:1874670011462528:192.168.203.104@tcp:35:lfs.0 00000002:00100000:0.0:1787826381.760162:0:69379:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d90844f00 x1874670011461632/t8589943019(8589943019) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 536/608 e 0 to 0 dl 1787826399 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826381.790577:0:69380:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826381.790622:0:69380:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826381.790641:0:69380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1787826381.790657:0:69380:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:0.0:1787826381.790665:0:69380:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826381.790674:0:69380:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826381.790677:0:69380:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:0.0:1787826381.790684:0:69380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142b37bf8 00010000:00010000:0.0:1787826381.790688:0:69380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:0.0:1787826381.790696:0:69380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:0.0:1787826381.790710:0:69380:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826381.790729:0:69380:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=27 00000080:00200000:0.0:1787826381.790735:0:69380:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:0.0:1787826381.790747:0:69380:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), file ffff907d8500af00, open_flags 18800, mode 0 opened 0 00000080:00200000:0.0:1787826381.791697:0:69380:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=open 00000002:00010000:0.0:1787826381.791729:0:69380:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x208:0x0], intent: open flags 0304001 00010000:00010000:0.0:1787826381.791828:0:69380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba9c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.791837:0:69380:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba9c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826381.791849:0:69380:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826381.791859:0:69380:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90845680 x1874670011462784/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826381.791869:0:69380:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba9c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826381.791888:0:69380:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69380:1874670011462784:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826381.792069:0:69380:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238a80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826381.802534:0:69380:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69380:1874670011462784:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826381.802572:0:69380:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba9c lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.802586:0:69380:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba9c lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.802594:0:69380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba9c lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.802601:0:69380:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba9c lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.802642:0:69380:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba9c lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.802648:0:69380:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883b600 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826381.802652:0:69380:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba9c lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.802659:0:69380:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba9c lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.802667:0:69380:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba9c lrc: 5/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.802674:0:69380:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826381.802681:0:69380:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba9c lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.802687:0:69380:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357ba9c lrc: 0/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826381.802725:0:69380:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d90845680 x1874670011462784/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826399 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826381.802741:0:69380:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x208:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1787826381.802898:0:69380:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826381.802918:0:69380:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826381.802932:0:69380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1787826381.802943:0:69380:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:0.0:1787826381.802950:0:69380:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826381.802956:0:69380:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826381.802959:0:69380:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:0.0:1787826381.802964:0:69380:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142b37bf8 00010000:00010000:0.0:1787826381.802967:0:69380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:0.0:1787826381.802975:0:69380:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:0.0:1787826381.802988:0:69380:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826381.802996:0:69380:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=267 00000080:00200000:0.0:1787826381.803002:0:69380:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 44 00000080:00200008:0.0:1787826381.803010:0:69380:0:(file.c:1090:ll_file_open()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00000002:00010000:0.0:1787826381.803024:0:69380:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x208:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:0.0:1787826381.803066:0:69380:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357baa3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.803073:0:69380:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357baa3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826381.803082:0:69380:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826381.803088:0:69380:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90845680 x1874670011463040/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826381.803097:0:69380:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357baa3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826381.803115:0:69380:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69380:1874670011463040:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826381.803281:0:69380:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238a80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826381.808932:0:69380:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69380:1874670011463040:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826381.808956:0:69380:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357baa3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.808968:0:69380:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357baa3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.808975:0:69380:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357baa3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.808982:0:69380:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357baa3 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.808990:0:69380:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357baa3 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.809000:0:69380:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a600 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826381.809005:0:69380:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357baa3 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.809012:0:69380:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357baa3 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.809019:0:69380:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357baa3 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.809026:0:69380:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826381.809038:0:69380:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357baa3 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826381.809045:0:69380:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a600/0xe5f736940357baa3 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69380 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826381.809064:0:69380:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffff907d90845680 x1874670011463040/t8589943023(8589943023) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826399 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1787826381.809078:0:69380:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90845680 x1874670011463040/t8589943023(8589943023) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826399 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826381.809088:0:69380:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1787826381.809108:0:69380:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x208:0x0], updating i_size 4096 00000002:00100000:0.0:1787826381.809126:0:69380:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90845680 x1874670011463040/t8589943023(8589943023) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826399 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:0.0:1787826381.809224:0:69380:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8) cmd=c0086915 arg=25613d0 00000100:00100000:0.0:1787826381.809282:0:69380:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69380:1874670011463168:192.168.203.104@tcp:34:lfs.0 00000100:00100000:0.0:1787826381.809373:0:69380:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238a80 going to sleep for 13 seconds 00000100:00100000:0.0:1787826381.815264:0:69380:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69380:1874670011463168:192.168.203.104@tcp:34:lfs.0 00000080:00200008:0.0:1787826381.815443:0:69380:0:(file.c:479:ll_file_release()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00000002:00100000:0.0:1787826381.815501:0:69380:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d90845680 x1874670011463040/t8589943023(8589943023) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826399 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1787826381.815525:0:69380:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1787826381.815531:0:69380:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90844000 x1874670011463424/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1787826381.815552:0:69380:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69380:1874670011463424:192.168.203.104@tcp:35:lfs.0 00000100:00100000:0.0:1787826381.815657:0:69380:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238a80 going to sleep for 11 seconds 00000100:00100000:0.0:1787826381.828981:0:69380:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69380:1874670011463424:192.168.203.104@tcp:35:lfs.0 00000002:00100000:0.0:1787826381.829015:0:69380:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d90845680 x1874670011463040/t8589943023(8589943023) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826399 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1787826381.882467:0:69384:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1787826381.882479:0:69384:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826381.882499:0:69384:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826381.882518:0:69384:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1787826381.882540:0:69384:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1787826381.882548:0:69384:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826381.882555:0:69384:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826381.882558:0:69384:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:2.0:1787826381.882564:0:69384:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142b37cb8 00010000:00010000:2.0:1787826381.882568:0:69384:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1787826381.882577:0:69384:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1787826381.882673:0:69384:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826381.882680:0:69384:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826381.882687:0:69384:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1787826381.882694:0:69384:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1787826381.882700:0:69384:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826381.882704:0:69384:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826381.882707:0:69384:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:2.0:1787826381.882711:0:69384:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142b37bb0 00010000:00010000:2.0:1787826381.882723:0:69384:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1787826381.882731:0:69384:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:2.0:1787826381.882740:0:69384:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:2.0:1787826381.882746:0:69384:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:2.0:1787826381.882824:0:69384:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffc66d9fe8c 00000080:00200008:2.0:1787826381.882837:0:69384:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:2.0:1787826381.882859:0:69384:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826381.882866:0:69384:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826381.882873:0:69384:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1787826381.882882:0:69384:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:2.0:1787826381.882888:0:69384:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826381.882893:0:69384:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826381.882895:0:69384:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:2.0:1787826381.882899:0:69384:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142b37bb0 00010000:00010000:2.0:1787826381.882902:0:69384:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1787826381.882910:0:69384:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:2.0:1787826381.882919:0:69384:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:2.0:1787826381.882923:0:69384:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 100000 00000080:00200080:2.0:1787826381.882932:0:69384:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=8080667f arg=7ffc66d9eb60 00000080:00200008:2.0:1787826381.886544:0:69384:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 100000 00000080:00200008:2.0:1787826381.887046:0:69384:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1787826381.887053:0:69384:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826381.887073:0:69384:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826381.887089:0:69384:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1787826381.887103:0:69384:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:2.0:1787826381.887110:0:69384:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826381.887442:0:69384:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826381.887447:0:69384:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:2.0:1787826381.887455:0:69384:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142b37cb8 00010000:00010000:2.0:1787826381.887460:0:69384:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1787826381.887470:0:69384:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1787826382.714759:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826383.736842:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826384.634410:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2348:1874670011463552:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826384.634520:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011463808:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826384.634569:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1787826384.634576:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011463680:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826384.634603:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011463936:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826384.639994:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011463680:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826384.640010:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011463936:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826384.640019:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2348:1874670011463552:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826384.640029:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011463808:192.168.203.104@tcp:400:kworker.0 00010000:00010000:0.0:1787826384.759674:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826385.783681:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826386.805235:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826387.833110:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826388.852907:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826389.753844:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011464192:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826389.753988:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011464448:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826389.754030:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1787826389.754038:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011464064:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826389.754076:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011464320:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826389.759259:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011464064:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826389.759281:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011464320:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826389.759292:0:2349:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589943024 gen 1 00000100:00100000:1.0:1787826389.759354:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011464192:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826389.759370:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011464448:192.168.203.104@tcp:400:kworker.0 00010000:00010000:1.0:1787826389.876971:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826390.900836:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826391.925030:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826392.948770:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826393.972811:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1787826394.211897:0:69407:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826394.211929:0:69407:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826394.211950:0:69407:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1787826394.211973:0:69407:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:2.0:1787826394.211981:0:69407:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826394.211988:0:69407:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826394.211992:0:69407:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:2.0:1787826394.212000:0:69407:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142b37bf8 00010000:00010000:2.0:1787826394.212005:0:69407:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1787826394.212013:0:69407:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:2.0:1787826394.212029:0:69407:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826394.212038:0:69407:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=27 00000080:00200000:2.0:1787826394.212043:0:69407:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:2.0:1787826394.212053:0:69407:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), file ffff907d91edf380, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1787826394.212060:0:69407:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=open 00000002:00010000:2.0:1787826394.212074:0:69407:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x208:0x0], intent: open flags 0304001 00010000:00010000:2.0:1787826394.212127:0:69407:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357baaa lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826394.212135:0:69407:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357baaa lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826394.212148:0:69407:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826394.212162:0:69407:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d91148780 x1874670011464704/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826394.212173:0:69407:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357baaa lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826394.212196:0:69407:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69407:1874670011464704:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826394.219884:0:69407:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d872480c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826394.227398:0:69407:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69407:1874670011464704:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826394.227423:0:69407:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357baaa lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826394.227437:0:69407:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357baaa lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826394.227444:0:69407:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357baaa lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826394.227466:0:69407:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357baaa lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826394.227474:0:69407:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357baaa lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826394.227479:0:69407:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d863c1400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826394.227483:0:69407:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357baaa lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826394.227489:0:69407:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357baaa lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826394.227496:0:69407:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357baaa lrc: 5/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826394.227502:0:69407:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826394.227509:0:69407:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357baaa lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826394.227514:0:69407:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d863c1400/0xe5f736940357baaa lrc: 0/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826394.227534:0:69407:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d91148780 x1874670011464704/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826412 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826394.227547:0:69407:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x208:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1787826394.227733:0:69407:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826394.227753:0:69407:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826394.227768:0:69407:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:0.0:1787826394.227780:0:69407:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:0.0:1787826394.227788:0:69407:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826394.227793:0:69407:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826394.227796:0:69407:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:0.0:1787826394.227801:0:69407:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142b37bf8 00010000:00010000:0.0:1787826394.227805:0:69407:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:0.0:1787826394.227813:0:69407:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:0.0:1787826394.227826:0:69407:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826394.227834:0:69407:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=267 00000080:00200000:0.0:1787826394.227840:0:69407:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 44 00000080:00200008:0.0:1787826394.227847:0:69407:0:(file.c:1090:ll_file_open()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00000002:00010000:0.0:1787826394.227862:0:69407:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x208:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:0.0:1787826394.227906:0:69407:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357bab1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826394.227914:0:69407:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357bab1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826394.227924:0:69407:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826394.227930:0:69407:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90845a40 x1874670011464960/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826394.227945:0:69407:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357bab1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826394.227965:0:69407:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69407:1874670011464960:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826394.228174:0:69407:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238600 going to sleep for 13 seconds 00000100:00100000:0.0:1787826394.237384:0:69407:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69407:1874670011464960:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826394.237408:0:69407:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357bab1 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826394.237421:0:69407:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357bab1 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826394.237429:0:69407:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357bab1 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826394.237437:0:69407:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357bab1 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826394.237445:0:69407:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357bab1 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826394.237450:0:69407:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883aa00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826394.237467:0:69407:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357bab1 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826394.237474:0:69407:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357bab1 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826394.237480:0:69407:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357bab1 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826394.237488:0:69407:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826394.237507:0:69407:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357bab1 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826394.237515:0:69407:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883aa00/0xe5f736940357bab1 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69407 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826394.237533:0:69407:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffff907d90845a40 x1874670011464960/t8589943025(8589943025) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826412 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1787826394.237546:0:69407:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90845a40 x1874670011464960/t8589943025(8589943025) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826412 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826394.237558:0:69407:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1787826394.237578:0:69407:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x208:0x0], updating i_size 4096 00000002:00100000:0.0:1787826394.237596:0:69407:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d90845a40 x1874670011464960/t8589943025(8589943025) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826412 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:0.0:1787826394.237726:0:69407:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8) cmd=c0086915 arg=95a3d0 00000100:00100000:0.0:1787826394.237790:0:69407:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69407:1874670011465088:192.168.203.104@tcp:34:lfs.0 00000100:00100000:0.0:1787826394.237873:0:69407:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238600 going to sleep for 13 seconds 00000100:00100000:0.0:1787826394.243404:0:69407:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69407:1874670011465088:192.168.203.104@tcp:34:lfs.0 00000080:00200008:0.0:1787826394.243709:0:69407:0:(file.c:479:ll_file_release()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00000002:00100000:0.0:1787826394.243770:0:69407:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d90845a40 x1874670011464960/t8589943025(8589943025) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826412 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1787826394.243782:0:69407:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1787826394.243788:0:69407:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908452c0 x1874670011465344/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1787826394.243811:0:69407:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69407:1874670011465344:192.168.203.104@tcp:35:lfs.0 00000100:00100000:0.0:1787826394.243898:0:69407:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238600 going to sleep for 11 seconds 00000100:00100000:0.0:1787826394.247172:0:69407:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69407:1874670011465344:192.168.203.104@tcp:35:lfs.0 00000002:00100000:0.0:1787826394.247207:0:69407:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d90845a40 x1874670011464960/t8589943025(8589943025) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826412 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826394.263534:0:69408:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826394.263562:0:69408:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826394.263580:0:69408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1787826394.263596:0:69408:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:1.0:1787826394.263604:0:69408:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826394.265841:0:69408:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826394.265846:0:69408:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:1.0:1787826394.265857:0:69408:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142b1fbf8 00010000:00010000:1.0:1787826394.265863:0:69408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:1.0:1787826394.265877:0:69408:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:1.0:1787826394.265894:0:69408:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826394.265905:0:69408:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=25 00000080:00200000:1.0:1787826394.265912:0:69408:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:1.0:1787826394.265929:0:69408:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 3 00000080:00200000:1.0:1787826394.265933:0:69408:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), file ffff907d82ca8300, open_flags 8041, mode 81b6 opened 0 00000080:00200000:1.0:1787826394.265941:0:69408:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=open|creat 00000002:00010000:1.0:1787826394.265965:0:69408:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204e.sanity-flr,[0x200000404:0x20a:0x0]) in obj [0x200000404:0x208:0x0], intent: open|creat flags 0100102 00010000:00010000:1.0:1787826394.266020:0:69408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bab8 lrc: 3/0,1 mode: --/CW res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69408 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.266033:0:69408:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bab8 lrc: 3/0,1 mode: --/CW res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69408 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826394.266043:0:69408:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826394.266054:0:69408:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a2940 x1874670011465600/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/67896 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826394.266063:0:69408:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bab8 lrc: 3/0,1 mode: --/CW res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69408 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826394.266084:0:69408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:69408:1874670011465600:192.168.203.104@tcp:101:dd.0 00000100:00100000:1.0:1787826394.267306:0:69408:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7db40 going to sleep for 13 seconds 00000100:00100000:1.0:1787826394.283438:0:69408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:69408:1874670011465600:192.168.203.104@tcp:101:dd.0 00010000:00010000:1.0:1787826394.283470:0:69408:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bab8 lrc: 4/0,1 mode: --/CW res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10748 expref: -99 pid: 69408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.283492:0:69408:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bab8 lrc: 4/0,1 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10748 expref: -99 pid: 69408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.283501:0:69408:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bab8 lrc: 4/0,1 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10748 expref: -99 pid: 69408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.283512:0:69408:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bab8 lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10748 expref: -99 pid: 69408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.283522:0:69408:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bab8 lrc: 4/0,1 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10748 expref: -99 pid: 69408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.283540:0:69408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bab8 lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10748 expref: -99 pid: 69408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.283548:0:69408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bab8 lrc: 5/1,1 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10748 expref: -99 pid: 69408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.283555:0:69408:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bab8 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10748 expref: -99 pid: 69408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826394.283566:0:69408:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x2b, status=0 req@ffff907d908a2940 x1874670011465600/t8589943027(8589943027) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/1056 e 0 to 0 dl 1787826412 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826394.283580:0:69408:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a2940 x1874670011465600/t8589943027(8589943027) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/1056 e 0 to 0 dl 1787826412 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826394.283592:0:69408:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 448 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bab8 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10748 expref: -99 pid: 69408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826394.283605:0:69408:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bab8 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10748 expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.283647:0:69408:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/521 (0 0) 00000080:00200000:1.0:1787826394.283653:0:69408:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x200000404:0x208:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1787826394.283701:0:69408:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x209:0x0](ffff907d933121c8): rc = 0 00000080:00200000:1.0:1787826394.283715:0:69408:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x209:0x0]: layout version change: 4294967294 -> 4 00000080:00010000:1.0:1787826394.283720:0:69408:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) for lock 0xe5f736940357bab8 00000080:00010000:1.0:1787826394.283731:0:69408:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) 00000080:00200008:1.0:1787826394.283745:0:69408:0:(file.c:1090:ll_file_open()) START file f204e.sanity-flr:[0x200000404:0x209:0x0](ffff907d933121c8), flags 100101 00000002:00100000:1.0:1787826394.283766:0:69408:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a2940 x1874670011465600/t8589943027(8589943027) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 984/1056 e 0 to 0 dl 1787826412 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1787826394.283800:0:69408:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357bab8 from it ffff907da56c27e0 00010000:00010000:1.0:1787826394.283805:0:69408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bab8 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10748 expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.283814:0:69408:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bab8 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10748 expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826394.283833:0:69408:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8), inode mode 81a4 mask 42 00000080:00200008:1.0:1787826394.293060:0:69408:0:(file.c:2652:do_file_write_iter()) START file f204e.sanity-flr:[0x200000404:0x209:0x0], ppos: 0, count: 2097152 00000080:00200000:1.0:1787826394.293075:0:69408:0:(file.c:2679:do_file_write_iter()) switching to DIO 00000080:00200000:1.0:1787826394.293082:0:69408:0:(file.c:2090:ll_file_io_generic()) f204e.sanity-flr: write ppos: 0, bytes: 2097152 00000080:00200000:1.0:1787826394.293096:0:69408:0:(file.c:1848:ll_io_set_mirror()) f204e.sanity-flr: desiginated mirror: 0 00000020:00200000:1.0:1787826394.293105:0:69408:0:(cl_io.c:218:cl_io_rw_init()) header@0000000033ed2c48[0x110, 2, [0x200000404:0x209:0x0] hash] 00000020:00200000:1.0:1787826394.293107:0:69408:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [0, 2097152) 0 0 00000080:00200000:1.0:1787826394.293113:0:69408:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:1.0:1787826394.293223:0:69408:0:(lov_io.c:571:lov_io_mirror_init()) [0x200000404:0x209:0x0] need write intent for [0, 2097152) 00000020:00200000:1.0:1787826394.293228:0:69408:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1787826394.293236:0:69408:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 4 need write layout 1, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826394.293239:0:69408:0:(vvp_io.c:371:vvp_io_fini()) [0x200000404:0x209:0x0] write layout, type 2 [0x0, 0x200000) 00000080:00010000:1.0:1787826394.293250:0:69408:0:(file.c:7076:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000404:0x209:0x0](ffff907d933121c8) 00000002:00010000:1.0:1787826394.293264:0:69408:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x209:0x0]) in obj [0x200000404:0x209:0x0], intent: layout flags 02 00010000:00010000:1.0:1787826394.293288:0:69408:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bab8 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc10748 expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826394.293309:0:69408:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x209:0x0]: losing layout lock 00010000:00010000:1.0:1787826394.293356:0:69408:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bab8 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc10748 expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826394.293364:0:69408:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826394.293367:0:69408:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bab8 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc10748 expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826394.293405:0:69408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 3/0,1 mode: --/EX res: [0x200000404:0x209:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 69408 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.293412:0:69408:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 3/0,1 mode: --/EX res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 69408 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826394.293424:0:69408:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826394.293438:0:69408:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a21c0 x1874670011465856/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/2048 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826394.293447:0:69408:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 3/0,1 mode: --/EX res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 69408 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826394.293467:0:69408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:69408:1874670011465856:192.168.203.104@tcp:101:dd.0 00000100:00100000:1.0:1787826394.293679:0:69408:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7db40 going to sleep for 13 seconds 00000100:00100000:1.0:1787826394.317553:0:69408:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589943028 gen 1 00000100:00100000:1.0:1787826394.317572:0:69408:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d908a2940 x1874670011465600/t8589943027(8589943027) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 984/1056 e 0 to 0 dl 1787826412 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1787826394.317585:0:69408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:69408:1874670011465856:192.168.203.104@tcp:101:dd.0 00010000:00010000:1.0:1787826394.317645:0:69408:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 4/0,1 mode: --/EX res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.317662:0:69408:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 4/0,1 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.317675:0:69408:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 4/0,1 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.317686:0:69408:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 4/0,1 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.317700:0:69408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 5/1,1 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.317709:0:69408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 5/1,1 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.317716:0:69408:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826394.317727:0:69408:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x400 disposition=0x1, status=0 req@ffff907d908a21c0 x1874670011465856/t8589943028(8589943028) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/896 e 0 to 0 dl 1787826412 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826394.317746:0:69408:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826394.317758:0:69408:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.317774:0:69408:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/521 (0 0) 00000080:00200000:1.0:1787826394.317780:0:69408:0:(namei.c:952:ll_intent_lock()) intent lock 1024 on i1 [0x200000404:0x209:0x0] suppgids 0 0: rc 0 00000080:00010000:1.0:1787826394.317799:0:69408:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) for lock 0xe5f736940357babf 00000080:00010000:1.0:1787826394.317807:0:69408:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357babf from it ffff9dd142b1fbb8 00010000:00010000:1.0:1787826394.317810:0:69408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.317819:0:69408:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826394.317835:0:69408:0:(file.c:2240:ll_file_io_generic()) f204e.sanity-flr: 2 io complete with rc: 0, result: 0, restart: 1 00000080:00200000:1.0:1787826394.317839:0:69408:0:(file.c:2247:ll_file_io_generic()) f204e.sanity-flr: restart write from ppos=0 bytes=2097152 retries=999 ret=0: rc = 0 00000080:00200000:1.0:1787826394.317846:0:69408:0:(file.c:1848:ll_io_set_mirror()) f204e.sanity-flr: desiginated mirror: 0 00000020:00200000:1.0:1787826394.317853:0:69408:0:(cl_io.c:218:cl_io_rw_init()) header@0000000033ed2c48[0x110, 2, [0x200000404:0x209:0x0] hash] 00000020:00200000:1.0:1787826394.317856:0:69408:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [0, 2097152) 0 0 00000080:00200000:1.0:1787826394.317861:0:69408:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 4 restore needed 0 00010000:00010000:1.0:1787826394.317879:0:69408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826394.317889:0:69408:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:1.0:1787826394.317898:0:69408:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x209:0x0](000000003c30eba3) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826394.317942:0:69408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826394.317955:0:69408:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826394.317968:0:69408:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826394.317978:0:69408:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826394.317982:0:69408:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826394.317989:0:69408:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1787826394.317998:0:69408:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:1.0:1787826394.318017:0:69408:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:1.0:1787826394.318029:0:69408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826394.318037:0:69408:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:1.0:1787826394.318043:0:69408:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x209:0x0](000000003c30eba3) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826394.318068:0:69408:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1787826394.318073:0:69408:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826394.318075:0:69408:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826394.318077:0:69408:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1787826394.318082:0:69408:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:1.0:1787826394.318088:0:69408:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826394.318181:0:69408:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x209:0x0]: layout version change: 4294967294 -> 6 00010000:00010000:1.0:1787826394.318186:0:69408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826394.318199:0:69408:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00020000:00200000:1.0:1787826394.318211:0:69408:0:(lov_io.c:311:lov_io_mirror_write_intent()) [0x200000404:0x209:0x0]there are 0 components to be staled to modify file extent [0x0, 0x6400000), iot: 2 00020000:00200000:1.0:1787826394.318215:0:69408:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826394.318217:0:69408:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00000080:00200000:1.0:1787826394.318221:0:69408:0:(file.c:2170:ll_file_io_generic()) Range lock [0, 511] 00020000:00200000:1.0:1787826394.318229:0:69408:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 2097152) 2097152, 2097152 00020000:00200000:1.0:1787826394.318231:0:69408:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1787826394.318243:0:69408:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 511] 00000008:00010000:1.0:1787826394.318253:0:69408:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1787826394.318258:0:69408:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c32fb0, osc lock ffff907d962f55f0, flags 0x0 00000080:00200000:1.0:1787826394.318266:0:69408:0:(vvp_io.c:1339:vvp_io_write_start()) f204e.sanity-flr: write [0, 2097152) append 0 append lockpos 0 00000002:00010000:1.0:1787826394.318292:0:69408:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x0:0x0:0x0]) in obj [0x200000404:0x209:0x0], intent: getxattr flags 0 00010000:00010000:1.0:1787826394.318331:0:69408:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bac6 lrc: 3/1,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69408 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.318339:0:69408:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bac6 lrc: 3/1,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69408 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.318348:0:69408:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bac6 lrc: 3/1,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69408 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826394.318367:0:69408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:69408:1874670011465984:192.168.203.104@tcp:101:dd.0 00000100:00100000:1.0:1787826394.318473:0:69408:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7db40 going to sleep for 13 seconds 00000100:00100000:1.0:1787826394.324489:0:69408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:69408:1874670011465984:192.168.203.104@tcp:101:dd.0 00010000:00010000:1.0:1787826394.324532:0:69408:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bac6 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1076b expref: -99 pid: 69408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.324559:0:69408:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bac6 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1076b expref: -99 pid: 69408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.324572:0:69408:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bac6 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1076b expref: -99 pid: 69408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826394.324653:0:69408:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x80 disposition=0x0, status=0 req@ffff907d908a21c0 x1874670011465984/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 568/1648 e 0 to 0 dl 1787826412 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826394.324669:0:69408:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bac6 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1076b expref: -99 pid: 69408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.324686:0:69408:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/521 (0 0) 00000080:00200000:1.0:1787826394.324695:0:69408:0:(namei.c:952:ll_intent_lock()) intent lock 128 on i1 [0x200000404:0x209:0x0] suppgids 0 -1: rc 0 00000080:00010000:1.0:1787826394.324730:0:69408:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) for lock 0xe5f736940357bac6 00000080:00010000:1.0:1787826394.324741:0:69408:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357bac6 from it ffff9dd142b1f8e8 00010000:00010000:1.0:1787826394.324746:0:69408:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bac6 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1076b expref: -99 pid: 69408 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826394.324757:0:69408:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bac6 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1076b expref: -99 pid: 69408 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826394.324825:0:69408:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8), size=2097152 (max 335544320), offset=0=0x0, pages 512 (max 81920), lockless, parallel, hybrid 00000020:00200000:2.0:1787826394.328754:0:69408:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 512 items remaining. 00000080:00200000:2.0:1787826394.328765:0:69408:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 2097152 count 2097152 file_offset 2097152 00000020:00010000:2.0:1787826394.328778:0:69408:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_release():245 00000020:00010000:2.0:1787826394.328782:0:69408:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826394.328786:0:69408:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c32fb0 (ffff907d884a71b8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826394.328789:0:69408:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826394.328792:0:69408:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c32fb0 (ffff907d884a7d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826394.328803:0:69408:0:(vvp_io.c:533:vvp_io_advance()) advancing 2097152 bytes 00000020:00200000:2.0:1787826394.328810:0:69408:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1787826394.328836:0:65304:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907da56f8f00 00000008:00100000:2.0:1787826394.330156:0:65304:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d91148f00 - object 0x240000400:809 offset 0<>2097152 00000100:00100000:3.0:1787826394.330318:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011466112:192.168.203.104@tcp:4:dd.0 00000100:00100000:3.0:1787826394.390382:0:2351:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff907d986d0000: committing for last_committed 21474840520 gen 3 00000008:00010000:3.0:1787826394.390524:0:2351:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907da56f8f00 00000020:00200000:3.0:1787826394.390533:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 511 items remaining. 00000020:00200000:3.0:1787826394.390537:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 510 items remaining. 00000020:00200000:3.0:1787826394.390539:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 509 items remaining. 00000020:00200000:3.0:1787826394.390540:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 508 items remaining. 00000020:00200000:3.0:1787826394.390543:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 507 items remaining. 00000020:00200000:3.0:1787826394.390545:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 506 items remaining. 00000020:00200000:3.0:1787826394.390546:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 505 items remaining. 00000020:00200000:3.0:1787826394.390548:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 504 items remaining. 00000020:00200000:3.0:1787826394.390550:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 503 items remaining. 00000020:00200000:3.0:1787826394.390552:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 502 items remaining. 00000020:00200000:3.0:1787826394.390553:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 501 items remaining. 00000020:00200000:3.0:1787826394.390554:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 500 items remaining. 00000020:00200000:3.0:1787826394.390556:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 499 items remaining. 00000020:00200000:3.0:1787826394.390557:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 498 items remaining. 00000020:00200000:3.0:1787826394.390559:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 497 items remaining. 00000020:00200000:3.0:1787826394.390560:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 496 items remaining. 00000020:00200000:3.0:1787826394.390562:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 495 items remaining. 00000020:00200000:3.0:1787826394.390563:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 494 items remaining. 00000020:00200000:3.0:1787826394.390564:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 493 items remaining. 00000020:00200000:3.0:1787826394.390565:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 492 items remaining. 00000020:00200000:3.0:1787826394.390567:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 491 items remaining. 00000020:00200000:3.0:1787826394.390568:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 490 items remaining. 00000020:00200000:3.0:1787826394.390570:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 489 items remaining. 00000020:00200000:3.0:1787826394.390572:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 488 items remaining. 00000020:00200000:3.0:1787826394.390573:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 487 items remaining. 00000020:00200000:3.0:1787826394.390575:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 486 items remaining. 00000020:00200000:3.0:1787826394.390576:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 485 items remaining. 00000020:00200000:3.0:1787826394.390578:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 484 items remaining. 00000020:00200000:3.0:1787826394.390580:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 483 items remaining. 00000020:00200000:3.0:1787826394.390582:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 482 items remaining. 00000020:00200000:3.0:1787826394.390583:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 481 items remaining. 00000020:00200000:3.0:1787826394.390585:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 480 items remaining. 00000020:00200000:3.0:1787826394.390586:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 479 items remaining. 00000020:00200000:3.0:1787826394.390588:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 478 items remaining. 00000020:00200000:3.0:1787826394.390590:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 477 items remaining. 00000020:00200000:3.0:1787826394.390591:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 476 items remaining. 00000020:00200000:3.0:1787826394.390594:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 475 items remaining. 00000020:00200000:3.0:1787826394.390596:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 474 items remaining. 00000020:00200000:3.0:1787826394.390597:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 473 items remaining. 00000020:00200000:3.0:1787826394.390599:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 472 items remaining. 00000020:00200000:3.0:1787826394.390601:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 471 items remaining. 00000020:00200000:3.0:1787826394.390602:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 470 items remaining. 00000020:00200000:3.0:1787826394.390604:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 469 items remaining. 00000020:00200000:3.0:1787826394.390663:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 468 items remaining. 00000020:00200000:3.0:1787826394.390666:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 467 items remaining. 00000020:00200000:3.0:1787826394.390667:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 466 items remaining. 00000020:00200000:3.0:1787826394.390669:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 465 items remaining. 00000020:00200000:3.0:1787826394.390671:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 464 items remaining. 00000020:00200000:3.0:1787826394.390673:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 463 items remaining. 00000020:00200000:3.0:1787826394.390675:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 462 items remaining. 00000020:00200000:3.0:1787826394.390677:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 461 items remaining. 00000020:00200000:3.0:1787826394.390679:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 460 items remaining. 00000020:00200000:3.0:1787826394.390682:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 459 items remaining. 00000020:00200000:3.0:1787826394.390684:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 458 items remaining. 00000020:00200000:3.0:1787826394.390686:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 457 items remaining. 00000020:00200000:3.0:1787826394.390689:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 456 items remaining. 00000020:00200000:3.0:1787826394.390691:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 455 items remaining. 00000020:00200000:3.0:1787826394.390693:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 454 items remaining. 00000020:00200000:3.0:1787826394.390695:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 453 items remaining. 00000020:00200000:3.0:1787826394.390697:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 452 items remaining. 00000020:00200000:3.0:1787826394.390702:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 451 items remaining. 00000020:00200000:3.0:1787826394.390704:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 450 items remaining. 00000020:00200000:3.0:1787826394.390707:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 449 items remaining. 00000020:00200000:3.0:1787826394.390708:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 448 items remaining. 00000020:00200000:3.0:1787826394.390713:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 447 items remaining. 00000020:00200000:3.0:1787826394.390717:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 446 items remaining. 00000020:00200000:3.0:1787826394.390719:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 445 items remaining. 00000020:00200000:3.0:1787826394.390721:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 444 items remaining. 00000020:00200000:3.0:1787826394.390724:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 443 items remaining. 00000020:00200000:3.0:1787826394.390725:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 442 items remaining. 00000020:00200000:3.0:1787826394.390727:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 441 items remaining. 00000020:00200000:3.0:1787826394.390729:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 440 items remaining. 00000020:00200000:3.0:1787826394.390731:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 439 items remaining. 00000020:00200000:3.0:1787826394.390733:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 438 items remaining. 00000020:00200000:3.0:1787826394.390735:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 437 items remaining. 00000020:00200000:3.0:1787826394.390737:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 436 items remaining. 00000020:00200000:3.0:1787826394.390738:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 435 items remaining. 00000020:00200000:3.0:1787826394.390740:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 434 items remaining. 00000020:00200000:3.0:1787826394.390742:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 433 items remaining. 00000020:00200000:3.0:1787826394.390744:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 432 items remaining. 00000020:00200000:3.0:1787826394.390746:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 431 items remaining. 00000020:00200000:3.0:1787826394.390747:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 430 items remaining. 00000020:00200000:3.0:1787826394.390749:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 429 items remaining. 00000020:00200000:3.0:1787826394.390751:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 428 items remaining. 00000020:00200000:3.0:1787826394.390753:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 427 items remaining. 00000020:00200000:3.0:1787826394.390755:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 426 items remaining. 00000020:00200000:3.0:1787826394.390756:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 425 items remaining. 00000020:00200000:3.0:1787826394.390758:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 424 items remaining. 00000020:00200000:3.0:1787826394.390760:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 423 items remaining. 00000020:00200000:3.0:1787826394.390762:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 422 items remaining. 00000020:00200000:3.0:1787826394.390764:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 421 items remaining. 00000020:00200000:3.0:1787826394.390766:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 420 items remaining. 00000020:00200000:3.0:1787826394.390769:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 419 items remaining. 00000020:00200000:3.0:1787826394.390770:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 418 items remaining. 00000020:00200000:3.0:1787826394.390772:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 417 items remaining. 00000020:00200000:3.0:1787826394.390774:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 416 items remaining. 00000020:00200000:3.0:1787826394.390775:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 415 items remaining. 00000020:00200000:3.0:1787826394.390777:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 414 items remaining. 00000020:00200000:3.0:1787826394.390778:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 413 items remaining. 00000020:00200000:3.0:1787826394.390780:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 412 items remaining. 00000020:00200000:3.0:1787826394.390782:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 411 items remaining. 00000020:00200000:3.0:1787826394.390784:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 410 items remaining. 00000020:00200000:3.0:1787826394.390785:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 409 items remaining. 00000020:00200000:3.0:1787826394.390787:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 408 items remaining. 00000020:00200000:3.0:1787826394.390788:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 407 items remaining. 00000020:00200000:3.0:1787826394.390790:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 406 items remaining. 00000020:00200000:3.0:1787826394.390792:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 405 items remaining. 00000020:00200000:3.0:1787826394.390794:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 404 items remaining. 00000020:00200000:3.0:1787826394.390796:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 403 items remaining. 00000020:00200000:3.0:1787826394.390798:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 402 items remaining. 00000020:00200000:3.0:1787826394.390800:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 401 items remaining. 00000020:00200000:3.0:1787826394.390802:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 400 items remaining. 00000020:00200000:3.0:1787826394.390803:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 399 items remaining. 00000020:00200000:3.0:1787826394.390805:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 398 items remaining. 00000020:00200000:3.0:1787826394.390807:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 397 items remaining. 00000020:00200000:3.0:1787826394.390809:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 396 items remaining. 00000020:00200000:3.0:1787826394.390811:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 395 items remaining. 00000020:00200000:3.0:1787826394.390813:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 394 items remaining. 00000020:00200000:3.0:1787826394.390815:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 393 items remaining. 00000020:00200000:3.0:1787826394.390816:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 392 items remaining. 00000020:00200000:3.0:1787826394.390818:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 391 items remaining. 00000020:00200000:3.0:1787826394.390819:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 390 items remaining. 00000020:00200000:3.0:1787826394.390822:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 389 items remaining. 00000020:00200000:3.0:1787826394.390823:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 388 items remaining. 00000020:00200000:3.0:1787826394.390826:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 387 items remaining. 00000020:00200000:3.0:1787826394.390827:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 386 items remaining. 00000020:00200000:3.0:1787826394.390829:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 385 items remaining. 00000020:00200000:3.0:1787826394.390830:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 384 items remaining. 00000020:00200000:3.0:1787826394.390832:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 383 items remaining. 00000020:00200000:3.0:1787826394.390834:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 382 items remaining. 00000020:00200000:3.0:1787826394.390836:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 381 items remaining. 00000020:00200000:3.0:1787826394.390838:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 380 items remaining. 00000020:00200000:3.0:1787826394.390841:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 379 items remaining. 00000020:00200000:3.0:1787826394.390843:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 378 items remaining. 00000020:00200000:3.0:1787826394.390845:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 377 items remaining. 00000020:00200000:3.0:1787826394.390847:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 376 items remaining. 00000020:00200000:3.0:1787826394.390849:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 375 items remaining. 00000020:00200000:3.0:1787826394.390850:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 374 items remaining. 00000020:00200000:3.0:1787826394.390852:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 373 items remaining. 00000020:00200000:3.0:1787826394.390853:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 372 items remaining. 00000020:00200000:3.0:1787826394.390855:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 371 items remaining. 00000020:00200000:3.0:1787826394.390857:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 370 items remaining. 00000020:00200000:3.0:1787826394.390858:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 369 items remaining. 00000020:00200000:3.0:1787826394.390860:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 368 items remaining. 00000020:00200000:3.0:1787826394.390862:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 367 items remaining. 00000020:00200000:3.0:1787826394.390864:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 366 items remaining. 00000020:00200000:3.0:1787826394.390866:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 365 items remaining. 00000020:00200000:3.0:1787826394.390868:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 364 items remaining. 00000020:00200000:3.0:1787826394.390870:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 363 items remaining. 00000020:00200000:3.0:1787826394.390872:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 362 items remaining. 00000020:00200000:3.0:1787826394.390874:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 361 items remaining. 00000020:00200000:3.0:1787826394.390876:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 360 items remaining. 00000020:00200000:3.0:1787826394.390878:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 359 items remaining. 00000020:00200000:3.0:1787826394.390880:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 358 items remaining. 00000020:00200000:3.0:1787826394.390882:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 357 items remaining. 00000020:00200000:3.0:1787826394.390884:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 356 items remaining. 00000020:00200000:3.0:1787826394.390886:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 355 items remaining. 00000020:00200000:3.0:1787826394.390888:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 354 items remaining. 00000020:00200000:3.0:1787826394.390889:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 353 items remaining. 00000020:00200000:3.0:1787826394.390891:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 352 items remaining. 00000020:00200000:3.0:1787826394.390893:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 351 items remaining. 00000020:00200000:3.0:1787826394.390895:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 350 items remaining. 00000020:00200000:3.0:1787826394.390896:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 349 items remaining. 00000020:00200000:3.0:1787826394.390898:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 348 items remaining. 00000020:00200000:3.0:1787826394.390901:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 347 items remaining. 00000020:00200000:3.0:1787826394.390902:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 346 items remaining. 00000020:00200000:3.0:1787826394.390904:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 345 items remaining. 00000020:00200000:3.0:1787826394.390906:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 344 items remaining. 00000020:00200000:3.0:1787826394.390908:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 343 items remaining. 00000020:00200000:3.0:1787826394.390909:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 342 items remaining. 00000020:00200000:3.0:1787826394.390910:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 341 items remaining. 00000020:00200000:3.0:1787826394.390912:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 340 items remaining. 00000020:00200000:3.0:1787826394.390913:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 339 items remaining. 00000020:00200000:3.0:1787826394.390915:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 338 items remaining. 00000020:00200000:3.0:1787826394.390917:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 337 items remaining. 00000020:00200000:3.0:1787826394.390919:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 336 items remaining. 00000020:00200000:3.0:1787826394.390921:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 335 items remaining. 00000020:00200000:3.0:1787826394.390922:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 334 items remaining. 00000020:00200000:3.0:1787826394.390924:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 333 items remaining. 00000020:00200000:3.0:1787826394.390926:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 332 items remaining. 00000020:00200000:3.0:1787826394.390928:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 331 items remaining. 00000020:00200000:3.0:1787826394.390931:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 330 items remaining. 00000020:00200000:3.0:1787826394.390933:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 329 items remaining. 00000020:00200000:3.0:1787826394.390935:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 328 items remaining. 00000020:00200000:3.0:1787826394.390937:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 327 items remaining. 00000020:00200000:3.0:1787826394.390938:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 326 items remaining. 00000020:00200000:3.0:1787826394.390940:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 325 items remaining. 00000020:00200000:3.0:1787826394.390942:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 324 items remaining. 00000020:00200000:3.0:1787826394.390944:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 323 items remaining. 00000020:00200000:3.0:1787826394.390946:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 322 items remaining. 00000020:00200000:3.0:1787826394.390947:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 321 items remaining. 00000020:00200000:3.0:1787826394.390949:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 320 items remaining. 00000020:00200000:3.0:1787826394.390951:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 319 items remaining. 00000020:00200000:3.0:1787826394.390952:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 318 items remaining. 00000020:00200000:3.0:1787826394.390954:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 317 items remaining. 00000020:00200000:3.0:1787826394.390956:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 316 items remaining. 00000020:00200000:3.0:1787826394.390958:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 315 items remaining. 00000020:00200000:3.0:1787826394.390960:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 314 items remaining. 00000020:00200000:3.0:1787826394.390962:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 313 items remaining. 00000020:00200000:3.0:1787826394.390964:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 312 items remaining. 00000020:00200000:3.0:1787826394.390965:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 311 items remaining. 00000020:00200000:3.0:1787826394.390967:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 310 items remaining. 00000020:00200000:3.0:1787826394.390968:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 309 items remaining. 00000020:00200000:3.0:1787826394.390970:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 308 items remaining. 00000020:00200000:3.0:1787826394.390971:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 307 items remaining. 00000020:00200000:3.0:1787826394.390973:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 306 items remaining. 00000020:00200000:3.0:1787826394.390975:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 305 items remaining. 00000020:00200000:3.0:1787826394.390976:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 304 items remaining. 00000020:00200000:3.0:1787826394.390978:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 303 items remaining. 00000020:00200000:3.0:1787826394.390979:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 302 items remaining. 00000020:00200000:3.0:1787826394.390981:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 301 items remaining. 00000020:00200000:3.0:1787826394.390983:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 300 items remaining. 00000020:00200000:3.0:1787826394.390984:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 299 items remaining. 00000020:00200000:3.0:1787826394.390986:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 298 items remaining. 00000020:00200000:3.0:1787826394.390988:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 297 items remaining. 00000020:00200000:3.0:1787826394.390990:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 296 items remaining. 00000020:00200000:3.0:1787826394.390992:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 295 items remaining. 00000020:00200000:3.0:1787826394.390993:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 294 items remaining. 00000020:00200000:3.0:1787826394.390995:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 293 items remaining. 00000020:00200000:3.0:1787826394.390997:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 292 items remaining. 00000020:00200000:3.0:1787826394.390999:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 291 items remaining. 00000020:00200000:3.0:1787826394.391001:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 290 items remaining. 00000020:00200000:3.0:1787826394.391003:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 289 items remaining. 00000020:00200000:3.0:1787826394.391004:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 288 items remaining. 00000020:00200000:3.0:1787826394.391006:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 287 items remaining. 00000020:00200000:3.0:1787826394.391008:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 286 items remaining. 00000020:00200000:3.0:1787826394.391010:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 285 items remaining. 00000020:00200000:3.0:1787826394.391011:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 284 items remaining. 00000020:00200000:3.0:1787826394.391013:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 283 items remaining. 00000020:00200000:3.0:1787826394.391015:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 282 items remaining. 00000020:00200000:3.0:1787826394.391017:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 281 items remaining. 00000020:00200000:3.0:1787826394.391019:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 280 items remaining. 00000020:00200000:3.0:1787826394.391020:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 279 items remaining. 00000020:00200000:3.0:1787826394.391022:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 278 items remaining. 00000020:00200000:3.0:1787826394.391023:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 277 items remaining. 00000020:00200000:3.0:1787826394.391025:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 276 items remaining. 00000020:00200000:3.0:1787826394.391026:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 275 items remaining. 00000020:00200000:3.0:1787826394.391028:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 274 items remaining. 00000020:00200000:3.0:1787826394.391029:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 273 items remaining. 00000020:00200000:3.0:1787826394.391031:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 272 items remaining. 00000020:00200000:3.0:1787826394.391033:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 271 items remaining. 00000020:00200000:3.0:1787826394.391035:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 270 items remaining. 00000020:00200000:3.0:1787826394.391037:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 269 items remaining. 00000020:00200000:3.0:1787826394.391039:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 268 items remaining. 00000020:00200000:3.0:1787826394.391042:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 267 items remaining. 00000020:00200000:3.0:1787826394.391043:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 266 items remaining. 00000020:00200000:3.0:1787826394.391045:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 265 items remaining. 00000020:00200000:3.0:1787826394.391047:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 264 items remaining. 00000020:00200000:3.0:1787826394.391049:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 263 items remaining. 00000020:00200000:3.0:1787826394.391051:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 262 items remaining. 00000020:00200000:3.0:1787826394.391053:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 261 items remaining. 00000020:00200000:3.0:1787826394.391055:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 260 items remaining. 00000020:00200000:3.0:1787826394.391057:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 259 items remaining. 00000020:00200000:3.0:1787826394.391059:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 258 items remaining. 00000020:00200000:3.0:1787826394.391061:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 257 items remaining. 00000020:00200000:3.0:1787826394.391063:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000020:00200000:3.0:1787826394.391064:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:3.0:1787826394.391066:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:3.0:1787826394.391068:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:3.0:1787826394.391069:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:3.0:1787826394.391071:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:3.0:1787826394.391073:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:3.0:1787826394.391074:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:3.0:1787826394.391076:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:3.0:1787826394.391077:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:3.0:1787826394.391079:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:3.0:1787826394.391080:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:3.0:1787826394.391082:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:3.0:1787826394.391084:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:3.0:1787826394.391086:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:3.0:1787826394.391088:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:3.0:1787826394.391089:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:3.0:1787826394.391091:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:3.0:1787826394.391093:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:3.0:1787826394.391095:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:3.0:1787826394.391097:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:3.0:1787826394.391098:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:3.0:1787826394.391100:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:3.0:1787826394.391102:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:3.0:1787826394.391104:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:3.0:1787826394.391105:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:3.0:1787826394.391107:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:3.0:1787826394.391110:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:3.0:1787826394.391111:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:3.0:1787826394.391113:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:3.0:1787826394.391115:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:3.0:1787826394.391117:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:3.0:1787826394.391118:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:3.0:1787826394.391120:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:3.0:1787826394.391122:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:3.0:1787826394.391124:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:3.0:1787826394.391126:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:3.0:1787826394.391129:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:3.0:1787826394.391130:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:3.0:1787826394.391132:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:3.0:1787826394.391134:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:3.0:1787826394.391136:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:3.0:1787826394.391138:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:3.0:1787826394.391139:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:3.0:1787826394.391141:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:3.0:1787826394.391143:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:3.0:1787826394.391144:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:3.0:1787826394.391147:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:3.0:1787826394.391149:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:3.0:1787826394.391151:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:3.0:1787826394.391153:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:3.0:1787826394.391155:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:3.0:1787826394.391157:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:3.0:1787826394.391159:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:3.0:1787826394.391160:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:3.0:1787826394.391162:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:3.0:1787826394.391164:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:3.0:1787826394.391166:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:3.0:1787826394.391167:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:3.0:1787826394.391169:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:3.0:1787826394.391171:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:3.0:1787826394.391173:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:3.0:1787826394.391174:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:3.0:1787826394.391176:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:3.0:1787826394.391177:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:3.0:1787826394.391179:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:3.0:1787826394.391180:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:3.0:1787826394.391182:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:3.0:1787826394.391184:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:3.0:1787826394.391186:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:3.0:1787826394.391188:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:3.0:1787826394.391190:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:3.0:1787826394.391192:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:3.0:1787826394.391194:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:3.0:1787826394.391195:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:3.0:1787826394.391197:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:3.0:1787826394.391199:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:3.0:1787826394.391201:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:3.0:1787826394.391203:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:3.0:1787826394.391205:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:3.0:1787826394.391207:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:3.0:1787826394.391209:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:3.0:1787826394.391211:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:3.0:1787826394.391212:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:3.0:1787826394.391214:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:3.0:1787826394.391216:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:3.0:1787826394.391218:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:3.0:1787826394.391219:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:3.0:1787826394.391221:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:3.0:1787826394.391223:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:3.0:1787826394.391224:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:3.0:1787826394.391226:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:3.0:1787826394.391228:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:3.0:1787826394.391230:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:3.0:1787826394.391232:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:3.0:1787826394.391234:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:3.0:1787826394.391236:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:3.0:1787826394.391238:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:3.0:1787826394.391240:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:3.0:1787826394.391241:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:3.0:1787826394.391243:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:3.0:1787826394.391245:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:3.0:1787826394.391247:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:3.0:1787826394.391249:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:3.0:1787826394.391251:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:3.0:1787826394.391253:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:3.0:1787826394.391255:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:3.0:1787826394.391257:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:3.0:1787826394.391258:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:3.0:1787826394.391260:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:3.0:1787826394.391262:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:3.0:1787826394.391263:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:3.0:1787826394.391265:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:3.0:1787826394.391267:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:3.0:1787826394.391269:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:3.0:1787826394.391271:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:3.0:1787826394.391273:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:3.0:1787826394.391274:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:3.0:1787826394.391277:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:3.0:1787826394.391279:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:3.0:1787826394.391281:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:3.0:1787826394.391283:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:3.0:1787826394.391284:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:3.0:1787826394.391286:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:3.0:1787826394.391288:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:3.0:1787826394.391290:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:3.0:1787826394.391292:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:3.0:1787826394.391293:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:3.0:1787826394.391295:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:3.0:1787826394.391297:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:3.0:1787826394.391299:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:3.0:1787826394.391300:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:3.0:1787826394.391302:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:3.0:1787826394.391305:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:3.0:1787826394.391307:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:3.0:1787826394.391309:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:3.0:1787826394.391311:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:3.0:1787826394.391312:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:3.0:1787826394.391314:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:3.0:1787826394.391316:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:3.0:1787826394.391317:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:3.0:1787826394.391319:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:3.0:1787826394.391321:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:3.0:1787826394.391323:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:3.0:1787826394.391325:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:3.0:1787826394.391326:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:3.0:1787826394.391328:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:3.0:1787826394.391330:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:3.0:1787826394.391332:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:3.0:1787826394.391333:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:3.0:1787826394.391335:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:3.0:1787826394.391337:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:3.0:1787826394.391339:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:3.0:1787826394.391341:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:3.0:1787826394.391343:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:3.0:1787826394.391345:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:3.0:1787826394.391347:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:3.0:1787826394.391348:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:3.0:1787826394.391350:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:3.0:1787826394.391352:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:3.0:1787826394.391354:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:3.0:1787826394.391355:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:3.0:1787826394.391357:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:3.0:1787826394.391358:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:3.0:1787826394.391360:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:3.0:1787826394.391362:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:3.0:1787826394.391365:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:3.0:1787826394.391366:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:3.0:1787826394.391368:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:3.0:1787826394.391369:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:3.0:1787826394.391371:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:3.0:1787826394.391373:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:3.0:1787826394.391375:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:3.0:1787826394.391376:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:3.0:1787826394.391378:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:3.0:1787826394.391380:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:3.0:1787826394.391382:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:3.0:1787826394.391384:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:3.0:1787826394.391386:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:3.0:1787826394.391388:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:3.0:1787826394.391390:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:3.0:1787826394.391392:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:3.0:1787826394.391393:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:3.0:1787826394.391395:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:3.0:1787826394.391397:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:3.0:1787826394.391399:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:3.0:1787826394.391401:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:3.0:1787826394.391402:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:3.0:1787826394.391404:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:3.0:1787826394.391405:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:3.0:1787826394.391407:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:3.0:1787826394.391409:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:3.0:1787826394.391411:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:3.0:1787826394.391413:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:3.0:1787826394.391414:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:3.0:1787826394.391415:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:3.0:1787826394.391417:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:3.0:1787826394.391419:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:3.0:1787826394.391421:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:3.0:1787826394.391423:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:3.0:1787826394.391425:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:3.0:1787826394.391426:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:3.0:1787826394.391428:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:3.0:1787826394.391429:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:3.0:1787826394.391431:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:3.0:1787826394.391432:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:3.0:1787826394.391434:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:3.0:1787826394.391436:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:3.0:1787826394.391438:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:3.0:1787826394.391439:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:3.0:1787826394.391441:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:3.0:1787826394.391443:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:3.0:1787826394.391445:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:3.0:1787826394.391447:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:3.0:1787826394.391449:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:3.0:1787826394.391450:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:3.0:1787826394.391452:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:3.0:1787826394.391453:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:3.0:1787826394.391455:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:3.0:1787826394.391456:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:3.0:1787826394.391458:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:3.0:1787826394.391460:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:3.0:1787826394.391461:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:3.0:1787826394.391463:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:3.0:1787826394.391465:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:3.0:1787826394.391467:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:3.0:1787826394.391468:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:3.0:1787826394.391470:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:3.0:1787826394.391473:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:3.0:1787826394.391475:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:3.0:1787826394.391477:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:3.0:1787826394.391479:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:3.0:1787826394.391481:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:3.0:1787826394.391483:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:3.0:1787826394.391484:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:3.0:1787826394.391486:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:3.0:1787826394.391488:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:3.0:1787826394.391489:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:3.0:1787826394.391491:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:3.0:1787826394.391493:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:3.0:1787826394.391495:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:3.0:1787826394.391497:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:3.0:1787826394.391498:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:3.0:1787826394.391500:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:3.0:1787826394.391502:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:3.0:1787826394.391504:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:3.0:1787826394.391506:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:3.0:1787826394.391508:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:3.0:1787826394.391510:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:3.0:1787826394.391512:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:3.0:1787826394.391514:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:3.0:1787826394.391515:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:3.0:1787826394.391517:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:3.0:1787826394.391519:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:3.0:1787826394.391522:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:3.0:1787826394.391523:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:3.0:1787826394.391525:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:3.0:1787826394.391911:0:2351:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:3.0:1787826394.391941:0:2351:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907da56f8f00 00000100:00100000:3.0:1787826394.391985:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011466112:192.168.203.104@tcp:4:dd.0 00000080:00200000:2.0:1787826394.394871:0:69408:0:(file.c:2207:ll_file_io_generic()) Range unlock [0, 511] 00000080:00200000:2.0:1787826394.394897:0:69408:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 6 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826394.394902:0:69408:0:(file.c:2240:ll_file_io_generic()) f204e.sanity-flr: 2 io complete with rc: 0, result: 2097152, restart: 0 00000080:00200000:2.0:1787826394.394919:0:69408:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x209:0x0] updating i_size 2097152 i_blocks 4096 00000020:00200000:2.0:1787826394.394924:0:69408:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826394.394938:0:69408:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 2097152 00000080:00200008:2.0:1787826394.394981:0:69408:0:(file.c:479:ll_file_release()) START file f204e.sanity-flr:[0x200000404:0x209:0x0](ffff907d933121c8), flags 100001 00000002:00100000:2.0:1787826394.395051:0:69408:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d908a2940 x1874670011465600/t8589943027(8589943027) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 984/1056 e 0 to 0 dl 1787826412 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1787826394.395064:0:69408:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1787826394.395069:0:69408:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114a940 x1874670011466368/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1787826394.395092:0:69408:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:69408:1874670011466368:192.168.203.104@tcp:35:dd.0 00000100:00100000:2.0:1787826394.395447:0:69408:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d87248780 going to sleep for 11 seconds 00000100:00100000:0.0:1787826394.407974:0:69408:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job dd:6f891bcb-15cd-41e0-96e2-db843619c1be:69408:1874670011466368:192.168.203.104@tcp:35:dd.0 00000002:00100000:0.0:1787826394.408028:0:69408:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d908a2940 x1874670011465600/t8589943027(8589943027) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 984/1056 e 0 to 0 dl 1787826412 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000100:00100000:3.0:1787826394.871066:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011466624:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826394.871199:0:2351:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1787826394.871207:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011466496:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826394.881001:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011466624:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826394.887658:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011466496:192.168.203.104@tcp:400:kworker.0 00010000:00010000:3.0:1787826394.998835:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826396.023369:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826397.046340:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826398.068959:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826399.094234:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826399.991884:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011466752:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826399.992530:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011467008:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826399.992580:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1787826399.992587:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011466880:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826399.992683:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011467136:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826400.004115:0:2350:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589943030 gen 1 00000100:00100000:3.0:1787826400.004143:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011466880:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826400.004175:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011467136:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826400.004188:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011466752:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826400.004200:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011467008:192.168.203.104@tcp:400:kworker.0 00010000:00010000:3.0:1787826400.117021:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826401.140791:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826402.164975:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826403.189173:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826404.212917:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826405.108925:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011467392:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826405.109060:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011467648:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826405.109093:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1787826405.109099:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011467264:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826405.109123:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011467520:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826405.113711:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011467264:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826405.130395:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011467392:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826405.130418:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011467648:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826405.132877:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011467520:192.168.203.104@tcp:400:kworker.0 00010000:00010000:3.0:1787826405.237069:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826406.261781:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1787826407.166603:0:69431:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826407.166718:0:69431:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826407.166743:0:69431:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1787826407.166762:0:69431:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:2.0:1787826407.166771:0:69431:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826407.166781:0:69431:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826407.166786:0:69431:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:2.0:1787826407.166794:0:69431:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142b47bf8 00010000:00010000:2.0:1787826407.166799:0:69431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1787826407.166811:0:69431:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:2.0:1787826407.166832:0:69431:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826407.166841:0:69431:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=27 00000080:00200000:2.0:1787826407.166848:0:69431:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:2.0:1787826407.166863:0:69431:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), file ffff907d83cd1080, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1787826407.166871:0:69431:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=open 00000002:00010000:2.0:1787826407.166965:0:69431:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x208:0x0], intent: open flags 0304001 00010000:00010000:2.0:1787826407.167115:0:69431:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bacd lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.167124:0:69431:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bacd lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826407.167137:0:69431:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826407.167152:0:69431:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d91149e00 x1874670011467904/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826407.167163:0:69431:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bacd lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826407.167191:0:69431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69431:1874670011467904:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826407.167471:0:69431:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d87248b40 going to sleep for 13 seconds 00000100:00100000:1.0:1787826407.176726:0:69431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69431:1874670011467904:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826407.176760:0:69431:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bacd lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.176778:0:69431:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bacd lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.176786:0:69431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bacd lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.176793:0:69431:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bacd lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.176803:0:69431:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bacd lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.176811:0:69431:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8646b800 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826407.176817:0:69431:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bacd lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.176825:0:69431:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bacd lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.176834:0:69431:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bacd lrc: 5/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.176850:0:69431:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826407.176861:0:69431:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bacd lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.176869:0:69431:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bacd lrc: 0/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826407.176893:0:69431:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d91149e00 x1874670011467904/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826425 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826407.176910:0:69431:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x208:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826407.177095:0:69431:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826407.177121:0:69431:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826407.177141:0:69431:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1787826407.177155:0:69431:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:1.0:1787826407.177164:0:69431:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826407.177171:0:69431:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826407.177175:0:69431:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:1.0:1787826407.177182:0:69431:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142b47bf8 00010000:00010000:1.0:1787826407.177190:0:69431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:1.0:1787826407.177199:0:69431:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:1.0:1787826407.177216:0:69431:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826407.177225:0:69431:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=267 00000080:00200000:1.0:1787826407.177234:0:69431:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 44 00000080:00200008:1.0:1787826407.177244:0:69431:0:(file.c:1090:ll_file_open()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00000002:00010000:1.0:1787826407.177263:0:69431:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x208:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:1.0:1787826407.177319:0:69431:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bad4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.177326:0:69431:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bad4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826407.177337:0:69431:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826407.177345:0:69431:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a1e00 x1874670011468160/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826407.177355:0:69431:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bad4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826407.177380:0:69431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69431:1874670011468160:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826407.177598:0:69431:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7d840 going to sleep for 13 seconds 00000100:00100000:1.0:1787826407.189823:0:69431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69431:1874670011468160:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826407.189930:0:69431:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bad4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.189950:0:69431:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bad4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.189958:0:69431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bad4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.189966:0:69431:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bad4 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.189976:0:69431:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bad4 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.189982:0:69431:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0ae00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826407.189988:0:69431:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bad4 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.189997:0:69431:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bad4 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.190004:0:69431:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bad4 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.190012:0:69431:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826407.190022:0:69431:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bad4 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.190028:0:69431:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bad4 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69431 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826407.190056:0:69431:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffff907d908a1e00 x1874670011468160/t8589943031(8589943031) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826425 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826407.190071:0:69431:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a1e00 x1874670011468160/t8589943031(8589943031) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826425 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826407.190088:0:69431:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826407.190114:0:69431:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x208:0x0], updating i_size 4096 00000002:00100000:1.0:1787826407.190136:0:69431:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a1e00 x1874670011468160/t8589943031(8589943031) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826425 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1787826407.190252:0:69431:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8) cmd=c0086915 arg=12803d0 00000100:00100000:1.0:1787826407.190332:0:69431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69431:1874670011468288:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826407.192782:0:69431:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7d840 going to sleep for 13 seconds 00000100:00100000:1.0:1787826407.201800:0:69431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69431:1874670011468288:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826407.202157:0:69431:0:(file.c:479:ll_file_release()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00000002:00100000:1.0:1787826407.202240:0:69431:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d908a1e00 x1874670011468160/t8589943031(8589943031) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826425 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826407.202253:0:69431:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826407.202259:0:69431:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a03c0 x1874670011468544/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826407.202287:0:69431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69431:1874670011468544:192.168.203.104@tcp:35:lfs.0 00000100:00100000:1.0:1787826407.202398:0:69431:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7d840 going to sleep for 11 seconds 00000100:00100000:1.0:1787826407.209084:0:69431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69431:1874670011468544:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826407.209131:0:69431:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d908a1e00 x1874670011468160/t8589943031(8589943031) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826425 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:3.0:1787826407.227021:0:69433:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1787826407.227035:0:69433:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826407.227064:0:69433:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826407.227088:0:69433:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:3.0:1787826407.227107:0:69433:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:3.0:1787826407.227117:0:69433:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826407.227127:0:69433:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826407.227131:0:69433:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:3.0:1787826407.227141:0:69433:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142b9fcb8 00010000:00010000:3.0:1787826407.227145:0:69433:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00010000:00010000:3.0:1787826407.227154:0:69433:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:3.0:1787826407.227283:0:69433:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826407.227292:0:69433:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826407.227299:0:69433:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826407.227308:0:69433:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00000080:00200000:3.0:1787826407.227314:0:69433:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826407.227319:0:69433:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826407.227322:0:69433:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:3.0:1787826407.227326:0:69433:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142b9fbb0 00010000:00010000:3.0:1787826407.227329:0:69433:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 18 lru_type: 1 00010000:00010000:3.0:1787826407.227337:0:69433:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:3.0:1787826407.228453:0:69433:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:3.0:1787826407.228474:0:69433:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:3.0:1787826407.228507:0:69433:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffc196ee32c 00000080:00200008:3.0:1787826407.228528:0:69433:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:1.0:1787826407.233716:0:69437:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826407.233758:0:69437:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826407.233782:0:69437:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:1.0:1787826407.233801:0:69437:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00000080:00200000:1.0:1787826407.233811:0:69437:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826407.233822:0:69437:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826407.233827:0:69437:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:1.0:1787826407.233835:0:69437:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142bb7bf8 00010000:00010000:1.0:1787826407.233840:0:69437:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 19 lru_type: 1 00010000:00010000:1.0:1787826407.233850:0:69437:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:1.0:1787826407.233869:0:69437:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826407.233878:0:69437:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=27 00000080:00200000:1.0:1787826407.233885:0:69437:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:1.0:1787826407.233900:0:69437:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), file ffff907d99bc2180, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1787826407.233907:0:69437:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=open 00000002:00010000:1.0:1787826407.233924:0:69437:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x208:0x0], intent: open flags 0304001 00010000:00010000:1.0:1787826407.233993:0:69437:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357badb lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69437 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.234007:0:69437:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357badb lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69437 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826407.234019:0:69437:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826407.234034:0:69437:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a03c0 x1874670011468800/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826407.234044:0:69437:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357badb lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69437 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826407.234071:0:69437:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69437:1874670011468800:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826407.234281:0:69437:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7d840 going to sleep for 13 seconds 00000100:00100000:1.0:1787826407.260094:0:69437:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69437:1874670011468800:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826407.260139:0:69437:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357badb lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69437 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.260158:0:69437:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357badb lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69437 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.260166:0:69437:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357badb lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69437 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.260175:0:69437:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357badb lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69437 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.260186:0:69437:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357badb lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69437 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.260194:0:69437:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0ae00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826407.260201:0:69437:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357badb lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 69437 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.260210:0:69437:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357badb lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 69437 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.260218:0:69437:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357badb lrc: 5/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 69437 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.260226:0:69437:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826407.260235:0:69437:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357badb lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69437 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.260243:0:69437:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357badb lrc: 0/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69437 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826407.260271:0:69437:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908a03c0 x1874670011468800/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826425 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826407.260288:0:69437:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x208:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826407.260660:0:69437:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826407.260685:0:69437:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826407.260703:0:69437:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:1.0:1787826407.260717:0:69437:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00000080:00200000:1.0:1787826407.260725:0:69437:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826407.260731:0:69437:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826407.260734:0:69437:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:1.0:1787826407.260741:0:69437:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142bb7bf8 00010000:00010000:1.0:1787826407.260745:0:69437:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 20 lru_type: 1 00010000:00010000:1.0:1787826407.260754:0:69437:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:1.0:1787826407.260771:0:69437:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826407.260780:0:69437:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=267 00000080:00200000:1.0:1787826407.260789:0:69437:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 44 00000080:00200008:1.0:1787826407.260800:0:69437:0:(file.c:1090:ll_file_open()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00000002:00010000:1.0:1787826407.260819:0:69437:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x208:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240304001 00010000:00010000:1.0:1787826407.260876:0:69437:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69437 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.260884:0:69437:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69437 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826407.260896:0:69437:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826407.260904:0:69437:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a03c0 x1874670011469056/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826407.260916:0:69437:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69437 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826407.260941:0:69437:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69437:1874670011469056:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826407.261151:0:69437:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7d840 going to sleep for 13 seconds 00000100:00100000:1.0:1787826407.269632:0:69437:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a03c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69437:1874670011469056:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826407.269680:0:69437:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc107cd expref: -99 pid: 69437 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.269693:0:69437:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc107cd expref: -99 pid: 69437 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.269708:0:69437:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x208:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc107cd expref: -99 pid: 69437 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.269720:0:69437:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x208:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc107cd expref: -99 pid: 69437 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826407.269737:0:69437:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x200002b, status=0 req@ffff907d908a03c0 x1874670011469056/t8589943033(8589943033) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826425 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826407.269753:0:69437:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a03c0 x1874670011469056/t8589943033(8589943033) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826425 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826407.269765:0:69437:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x208:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc107cd expref: -99 pid: 69437 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.269776:0:69437:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/520 (0 0) 00000080:00200000:1.0:1787826407.269783:0:69437:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826407.269810:0:69437:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x208:0x0], updating i_size 4096 00000080:00010000:1.0:1787826407.269815:0:69437:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x208:0x0](ffff907d933242c8) for lock 0xe5f736940357bae2 00000080:00010000:1.0:1787826407.269824:0:69437:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357bae2 from it ffff9dd142bb7bd0 00010000:00010000:1.0:1787826407.269829:0:69437:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x208:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc107cd expref: -99 pid: 69437 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.269840:0:69437:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x208:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc107cd expref: -99 pid: 69437 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:1.0:1787826407.269876:0:69437:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d908a03c0 x1874670011469056/t8589943033(8589943033) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826425 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1787826407.269987:0:69437:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8) cmd=c1406916 arg=17d5a20 00000100:00100000:1.0:1787826407.270066:0:69437:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69437:1874670011469184:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826407.270168:0:69437:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7d840 going to sleep for 13 seconds 00000100:00100000:1.0:1787826407.275921:0:69437:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69437:1874670011469184:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826407.276149:0:69437:0:(file.c:479:ll_file_release()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00010000:00010000:1.0:1787826407.276190:0:69437:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x208:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc107cd expref: -99 pid: 69437 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826407.284784:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:0.0:1787826407.320791:0:69440:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1787826407.320805:0:69440:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826407.320832:0:69440:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826407.320862:0:69440:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00010000:00010000:3.0:1787826407.321181:0:69440:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00000080:00200000:3.0:1787826407.321194:0:69440:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826407.321205:0:69440:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826407.321210:0:69440:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:3.0:1787826407.321219:0:69440:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142affcb8 00010000:00010000:3.0:1787826407.321223:0:69440:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 21 lru_type: 1 00010000:00010000:3.0:1787826407.321233:0:69440:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:3.0:1787826407.321374:0:69440:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826407.321384:0:69440:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826407.321394:0:69440:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:3.0:1787826407.321404:0:69440:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00000080:00200000:3.0:1787826407.321411:0:69440:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826407.321415:0:69440:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826407.321418:0:69440:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:3.0:1787826407.321423:0:69440:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142affbb0 00010000:00010000:3.0:1787826407.321426:0:69440:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 22 lru_type: 1 00010000:00010000:3.0:1787826407.321434:0:69440:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:3.0:1787826407.321444:0:69440:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:3.0:1787826407.321452:0:69440:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:3.0:1787826407.321475:0:69440:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffd24dfa8bc 00000080:00200008:3.0:1787826407.321490:0:69440:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:2.0:1787826407.325188:0:69442:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826407.325231:0:69442:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826407.325258:0:69442:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1787826407.325278:0:69442:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00000080:00200000:2.0:1787826407.325288:0:69442:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826407.325300:0:69442:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826407.325304:0:69442:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:2.0:1787826407.325313:0:69442:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142bcfbf8 00010000:00010000:2.0:1787826407.325318:0:69442:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 23 lru_type: 1 00010000:00010000:2.0:1787826407.325326:0:69442:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:2.0:1787826407.325345:0:69442:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826407.325354:0:69442:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=27 00000080:00200000:2.0:1787826407.325361:0:69442:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:2.0:1787826407.325375:0:69442:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), file ffff907d83cd0180, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1787826407.325383:0:69442:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=open 00000002:00010000:2.0:1787826407.325400:0:69442:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x208:0x0], intent: open flags 0304001 00010000:00010000:2.0:1787826407.325476:0:69442:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a400/0xe5f736940357bae9 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 69442 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.325484:0:69442:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a400/0xe5f736940357bae9 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69442 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826407.325499:0:69442:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826407.325516:0:69442:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114ad00 x1874670011469440/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826407.325526:0:69442:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a400/0xe5f736940357bae9 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69442 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826407.325564:0:69442:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69442:1874670011469440:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826407.325982:0:69442:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d87248b40 going to sleep for 13 seconds 00000100:00100000:1.0:1787826407.336086:0:69442:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69442:1874670011469440:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826407.336132:0:69442:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a400/0xe5f736940357bae9 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69442 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.336150:0:69442:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a400/0xe5f736940357bae9 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 69442 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.336157:0:69442:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a400/0xe5f736940357bae9 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 69442 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.336166:0:69442:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a400/0xe5f736940357bae9 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 69442 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.336177:0:69442:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a400/0xe5f736940357bae9 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 69442 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.336188:0:69442:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8646a400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826407.336194:0:69442:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a400/0xe5f736940357bae9 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 69442 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.336201:0:69442:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a400/0xe5f736940357bae9 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 69442 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.336208:0:69442:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a400/0xe5f736940357bae9 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 69442 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.336216:0:69442:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826407.336226:0:69442:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a400/0xe5f736940357bae9 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 69442 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.336232:0:69442:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a400/0xe5f736940357bae9 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 69442 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826407.336255:0:69442:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d9114ad00 x1874670011469440/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826425 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826407.336270:0:69442:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x208:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826407.336878:0:69442:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826407.336908:0:69442:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826407.336929:0:69442:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1787826407.336943:0:69442:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:1.0:1787826407.336951:0:69442:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826407.336958:0:69442:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826407.336962:0:69442:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:1.0:1787826407.336968:0:69442:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142bcfbf8 00010000:00010000:1.0:1787826407.336972:0:69442:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:1.0:1787826407.336981:0:69442:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:1.0:1787826407.336998:0:69442:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826407.337006:0:69442:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=267 00000080:00200000:1.0:1787826407.337014:0:69442:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 44 00000080:00200008:1.0:1787826407.337025:0:69442:0:(file.c:1090:ll_file_open()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00000080:00200080:1.0:1787826407.337069:0:69442:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8) cmd=c1406916 arg=1e4ba20 00000100:00100000:1.0:1787826407.337445:0:69442:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69442:1874670011469568:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826407.337775:0:69442:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7d840 going to sleep for 13 seconds 00000100:00100000:1.0:1787826407.351212:0:69442:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69442:1874670011469568:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826407.351447:0:69442:0:(file.c:479:ll_file_release()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00010000:00010000:1.0:1787826407.351495:0:69442:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x208:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc107cd expref: -99 pid: 69437 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1787826407.395983:0:69446:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826407.396011:0:69446:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826407.396030:0:69446:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:2.0:1787826407.396046:0:69446:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:2.0:1787826407.396054:0:69446:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826407.396061:0:69446:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826407.396065:0:69446:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:2.0:1787826407.396072:0:69446:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142bd7bf8 00010000:00010000:2.0:1787826407.396077:0:69446:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00010000:00010000:2.0:1787826407.396087:0:69446:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:2.0:1787826407.396102:0:69446:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826407.396111:0:69446:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=27 00000080:00200000:2.0:1787826407.396116:0:69446:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:2.0:1787826407.396129:0:69446:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), file ffff907d852aca80, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1787826407.396135:0:69446:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=open 00000002:00010000:2.0:1787826407.396148:0:69446:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x208:0x0], intent: open flags 0304001 00010000:00010000:2.0:1787826407.396201:0:69446:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646bc00/0xe5f736940357baf0 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 69446 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.396210:0:69446:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646bc00/0xe5f736940357baf0 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69446 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826407.396222:0:69446:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826407.396233:0:69446:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d91149680 x1874670011469824/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826407.396243:0:69446:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646bc00/0xe5f736940357baf0 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69446 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826407.396264:0:69446:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69446:1874670011469824:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826407.396771:0:69446:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d87248780 going to sleep for 13 seconds 00000100:00100000:3.0:1787826407.403186:0:69446:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69446:1874670011469824:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826407.403220:0:69446:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646bc00/0xe5f736940357baf0 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69446 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.403238:0:69446:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646bc00/0xe5f736940357baf0 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 69446 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.403245:0:69446:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646bc00/0xe5f736940357baf0 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 69446 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.403253:0:69446:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646bc00/0xe5f736940357baf0 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 69446 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.403263:0:69446:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646bc00/0xe5f736940357baf0 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 69446 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.403271:0:69446:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8646bc00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826407.403276:0:69446:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646bc00/0xe5f736940357baf0 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 69446 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.403288:0:69446:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646bc00/0xe5f736940357baf0 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 69446 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.403296:0:69446:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646bc00/0xe5f736940357baf0 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 69446 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.403304:0:69446:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826407.403312:0:69446:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646bc00/0xe5f736940357baf0 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 69446 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.403319:0:69446:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646bc00/0xe5f736940357baf0 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 69446 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826407.403355:0:69446:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d91149680 x1874670011469824/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826425 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826407.403371:0:69446:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x208:0x0] suppgids 0 -1: rc -20 00000080:00200000:3.0:1787826407.403577:0:69446:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826407.403603:0:69446:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826407.403648:0:69446:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:3.0:1787826407.403672:0:69446:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:3.0:1787826407.403681:0:69446:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826407.403689:0:69446:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826407.403694:0:69446:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:3.0:1787826407.403700:0:69446:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142bd7bf8 00010000:00010000:3.0:1787826407.403704:0:69446:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:3.0:1787826407.403713:0:69446:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:3.0:1787826407.403731:0:69446:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826407.403739:0:69446:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=267 00000080:00200000:3.0:1787826407.403747:0:69446:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 44 00000080:00200008:3.0:1787826407.403756:0:69446:0:(file.c:1090:ll_file_open()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00000080:00200080:3.0:1787826407.403840:0:69446:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8) cmd=c0086915 arg=16253f0 00000100:00100000:3.0:1787826407.403913:0:69446:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69446:1874670011469952:192.168.203.104@tcp:34:lfs.0 00000100:00100000:3.0:1787826407.404022:0:69446:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da0397240 going to sleep for 13 seconds 00000100:00100000:3.0:1787826407.416267:0:69446:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69446:1874670011469952:192.168.203.104@tcp:34:lfs.0 00000080:00200008:3.0:1787826407.417027:0:69446:0:(file.c:479:ll_file_release()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00010000:00010000:3.0:1787826407.417071:0:69446:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x208:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc107cd expref: -99 pid: 69437 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1787826407.452658:0:69448:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826407.452702:0:69448:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826407.452728:0:69448:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:3.0:1787826407.452748:0:69448:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:3.0:1787826407.452757:0:69448:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826407.452768:0:69448:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826407.452772:0:69448:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:3.0:1787826407.452781:0:69448:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142bdfbf8 00010000:00010000:3.0:1787826407.452786:0:69448:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00010000:00010000:3.0:1787826407.452796:0:69448:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00000080:00200000:3.0:1787826407.452814:0:69448:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826407.452824:0:69448:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=27 00000080:00200000:3.0:1787826407.452831:0:69448:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:3.0:1787826407.452846:0:69448:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), file ffff907d85563200, open_flags 18800, mode 0 opened 0 00000080:00200000:3.0:1787826407.452855:0:69448:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=open 00000002:00010000:3.0:1787826407.452873:0:69448:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x208:0x0], intent: open flags 0304001 00010000:00010000:3.0:1787826407.452941:0:69448:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629de00/0xe5f736940357baf7 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 69448 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.452950:0:69448:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629de00/0xe5f736940357baf7 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69448 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1787826407.452966:0:69448:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1787826407.452981:0:69448:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83525680 x1874670011470208/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1787826407.452992:0:69448:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629de00/0xe5f736940357baf7 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69448 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826407.453020:0:69448:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69448:1874670011470208:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826407.453238:0:69448:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da0397240 going to sleep for 13 seconds 00000100:00100000:3.0:1787826407.469262:0:69448:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69448:1874670011470208:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826407.469297:0:69448:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629de00/0xe5f736940357baf7 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69448 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.469314:0:69448:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629de00/0xe5f736940357baf7 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 69448 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.469322:0:69448:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629de00/0xe5f736940357baf7 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 69448 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.469340:0:69448:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629de00/0xe5f736940357baf7 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 69448 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.469351:0:69448:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629de00/0xe5f736940357baf7 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 69448 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.469358:0:69448:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8629de00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826407.469364:0:69448:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629de00/0xe5f736940357baf7 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 69448 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.469373:0:69448:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629de00/0xe5f736940357baf7 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 69448 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.469381:0:69448:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629de00/0xe5f736940357baf7 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 69448 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.469390:0:69448:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826407.469399:0:69448:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629de00/0xe5f736940357baf7 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 69448 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.469412:0:69448:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629de00/0xe5f736940357baf7 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 69448 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826407.469438:0:69448:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d83525680 x1874670011470208/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826425 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826407.469455:0:69448:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x208:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1787826407.469740:0:69448:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826407.469766:0:69448:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826407.469785:0:69448:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00010000:00010000:2.0:1787826407.469799:0:69448:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00000080:00200000:2.0:1787826407.469807:0:69448:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826407.469813:0:69448:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826407.469818:0:69448:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:2.0:1787826407.469824:0:69448:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142bdfbf8 00010000:00010000:2.0:1787826407.469828:0:69448:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 28 lru_type: 1 00010000:00010000:2.0:1787826407.469838:0:69448:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000080:00200000:2.0:1787826407.469854:0:69448:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826407.469861:0:69448:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=267 00000080:00200000:2.0:1787826407.469869:0:69448:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 44 00000080:00200008:2.0:1787826407.469879:0:69448:0:(file.c:1090:ll_file_open()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00000080:00200080:2.0:1787826407.469925:0:69448:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8) cmd=c0086915 arg=fab3f0 00000100:00100000:2.0:1787826407.469999:0:69448:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69448:1874670011470336:192.168.203.104@tcp:34:lfs.0 00000100:00100000:2.0:1787826407.470108:0:69448:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d872480c0 going to sleep for 13 seconds 00000100:00100000:0.0:1787826407.478699:0:69448:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69448:1874670011470336:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826407.478996:0:69448:0:(file.c:479:ll_file_release()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00010000:00010000:1.0:1787826407.479032:0:69448:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x208:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc107cd expref: -99 pid: 69437 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1787826407.494603:0:69451:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826407.494782:0:69451:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826407.494803:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:1.0:1787826407.494820:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00000080:00200000:1.0:1787826407.494829:0:69451:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826407.494837:0:69451:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826407.494840:0:69451:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:1.0:1787826407.494848:0:69451:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142bd7b58 00010000:00010000:1.0:1787826407.494853:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 29 lru_type: 1 00010000:00010000:1.0:1787826407.494862:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00000080:00200000:1.0:1787826407.494877:0:69451:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826407.494886:0:69451:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=25 00000080:00200000:1.0:1787826407.494892:0:69451:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:1.0:1787826407.494903:0:69451:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), flags=9 00000080:00200000:1.0:1787826407.494907:0:69451:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=getattr 00000002:00010000:1.0:1787826407.494920:0:69451:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x208:0x0], intent: getattr flags 01 00010000:00010000:1.0:1787826407.494966:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 69451 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.494975:0:69451:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69451 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.494985:0:69451:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69451 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826407.495011:0:69451:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011470464:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826407.495177:0:69451:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7d840 going to sleep for 13 seconds 00000100:00100000:1.0:1787826407.502884:0:69451:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011470464:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826407.502911:0:69451:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.502928:0:69451:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 4/1,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.502936:0:69451:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 4/1,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.502947:0:69451:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.502959:0:69451:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.502971:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.502979:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.502987:0:69451:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826407.503005:0:69451:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d908a1e00 x1874670011470464/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/1136 e 0 to 0 dl 1787826425 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826407.503016:0:69451:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826407.503030:0:69451:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.503041:0:69451:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/521 (0 0) 00000080:00200000:1.0:1787826407.503048:0:69451:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x208:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1787826407.503100:0:69451:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x209:0x0](ffff907d933121c8): rc = 0 00000080:00200000:1.0:1787826407.503118:0:69451:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x209:0x0]: layout version change: 6 -> 6 00000080:00010000:1.0:1787826407.503123:0:69451:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) for lock 0xe5f736940357bafe 00000080:00010000:1.0:1787826407.503134:0:69451:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) 00000080:00010000:1.0:1787826407.503138:0:69451:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357bafe from it ffff9dd142bd7b20 00010000:00010000:1.0:1787826407.503141:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.503150:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:1.0:1787826407.503184:0:69451:0:(file.c:6313:ll_getattr_dentry()) START file f204e.sanity-flr:[0x200000404:0x209:0x0](ffff907d933121c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826407.503191:0:69451:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8),name=f204e.sanity-flr 00000002:00010000:1.0:1787826407.503211:0:69451:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x209:0x0]) in obj [0x200000404:0x208:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826407.503220:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826407.503231:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826407.503238:0:69451:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x208:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826407.503243:0:69451:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) 00000080:00010000:1.0:1787826407.503246:0:69451:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) for lock 0xe5f736940357bafe 00000080:00010000:1.0:1787826407.503250:0:69451:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357bafe from it ffff9dd142bd7cb8 00010000:00010000:1.0:1787826407.503253:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826407.503260:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826407.503277:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1787826407.503288:0:69451:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826407.503291:0:69451:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1787826407.503295:0:69451:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x209:0x0] 00000008:00010000:1.0:1787826407.503312:0:69451:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1787826407.503315:0:69451:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c32230, osc lock ffff907d962f4750, flags 0x400000400001002 00010000:00010000:1.0:1787826407.503361:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.503372:0:69451:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->104857599] (req 0->104857599) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.503382:0:69451:0:(ldlm_request.c:1139:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff907d908a1e00 x1874670011470592/t0(0) o101->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000020:00200000:1.0:1787826407.503427:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:1.0:1787826407.503440:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1787826407.503457:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011470592:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826407.506533:0:2349:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->104857599] (req 0->104857599) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.506572:0:2349:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1088a expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.506587:0:2349:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1088a expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.506600:0:2349:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1088a expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826407.506653:0:2349:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc1088a expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1787826407.506665:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1787826407.506682:0:2349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1088a expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.506692:0:2349:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1088a expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.506705:0:2349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1088a expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.506714:0:2349:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1088a expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1787826407.506733:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011470592:192.168.203.104@tcp:101:lfs.0 00000080:00200000:2.0:1787826407.506827:0:69451:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x209:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826407.506834:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_release():245 00000020:00010000:2.0:1787826407.506837:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826407.506841:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c32230 (ffff907da5608d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826407.506845:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826407.506848:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c32230 (ffff907d884a7908/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826407.506860:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/1, layout version 6 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826407.506914:0:69451:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826407.506932:0:69451:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826407.506948:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00010000:00010000:2.0:1787826407.506961:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00000080:00200000:2.0:1787826407.506971:0:69451:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826407.506977:0:69451:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826407.506981:0:69451:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:2.0:1787826407.506987:0:69451:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142bd7bf8 00010000:00010000:2.0:1787826407.506991:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 30 lru_type: 1 00010000:00010000:2.0:1787826407.506999:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00000080:00200000:2.0:1787826407.507008:0:69451:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826407.507016:0:69451:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=25 00000080:00200000:2.0:1787826407.507022:0:69451:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:2.0:1787826407.507025:0:69451:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204e.sanity-flr, flags=265 00000080:00200000:2.0:1787826407.507030:0:69451:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8), inode mode 81a4 mask 46 00000080:00200008:2.0:1787826407.507038:0:69451:0:(file.c:1090:ll_file_open()) START file f204e.sanity-flr:[0x200000404:0x209:0x0](ffff907d933121c8), flags 154402 00000002:00010000:2.0:1787826407.507052:0:69451:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x209:0x0]) in obj [0x200000404:0x208:0x0], intent: open flags 0100240114403 00010000:00010000:2.0:1787826407.507100:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 69451 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.507108:0:69451:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69451 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826407.507119:0:69451:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826407.507127:0:69451:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114a580 x1874670011470848/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826407.507137:0:69451:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69451 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826407.507158:0:69451:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011470848:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826407.507372:0:69451:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d87248480 going to sleep for 13 seconds 00000100:00100000:1.0:1787826407.521699:0:69451:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011470848:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826407.521739:0:69451:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10898 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.521758:0:69451:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 4/1,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10898 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.521767:0:69451:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 4/1,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10898 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.521779:0:69451:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10898 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.521790:0:69451:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10898 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.521804:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10898 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.521812:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10898 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.521819:0:69451:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10898 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826407.521830:0:69451:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffff907d9114a580 x1874670011470848/t8589943034(8589943034) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1160 e 0 to 0 dl 1787826425 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1787826407.521843:0:69451:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d9114a580 x1874670011470848/t8589943034(8589943034) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1160 e 0 to 0 dl 1787826425 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826407.521856:0:69451:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10898 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826407.521869:0:69451:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10898 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.521881:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826407.521892:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826407.521901:0:69451:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10898 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.521907:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x7918213f4fc10898 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.521915:0:69451:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x7918213f4fc10898 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.521940:0:69451:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:1.0:1787826407.521947:0:69451:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x208:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1787826407.522009:0:69451:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x209:0x0]: layout version change: 6 -> 6 00000080:00010000:1.0:1787826407.522014:0:69451:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) for lock 0xe5f736940357bafe 00010000:00010000:3.0:1787826407.522125:0:60483:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1787826407.522133:0:60483:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7918213f4fc10898 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.522141:0:60483:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8646a000 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826407.522146:0:60483:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7918213f4fc10898 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.522154:0:60483:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7918213f4fc10898 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.522163:0:60483:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 3/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc10898 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.522179:0:60483:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826407.522217:0:60483:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 2/0,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc10898 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.522224:0:60483:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787826407.522260:0:60483:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 1/0,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc10898 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.522269:0:60483:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646a000/0xe5f736940357bb0c lrc: 0/0,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc10898 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.522286:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826407.522288:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826407.522290:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826407.522299:0:2351:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1787826407.522317:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011470976:192.168.203.104@tcp:103:ldlm_bl.0 00000080:00010000:1.0:1787826407.522920:0:69451:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357bafe from it ffff9dd142bd7bf0 00010000:00010000:1.0:1787826407.522930:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826407.522941:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000002:00100000:2.0:1787826407.523056:0:69451:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d9114a580 x1874670011470848/t8589943034(8589943034) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826425 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1787826407.523241:0:69451:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8), inode mode 81a4 mask 4 00000080:00200000:2.0:1787826407.523250:0:69451:0:(xattr.c:760:ll_xattr_get()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8), xattr lov 00000080:00200008:2.0:1787826407.523279:0:69451:0:(file.c:6313:ll_getattr_dentry()) START file f204e.sanity-flr:[0x200000404:0x209:0x0](ffff907d933121c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1787826407.523287:0:69451:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8),name=f204e.sanity-flr 00000002:00010000:2.0:1787826407.523307:0:69451:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x209:0x0]) in obj [0x200000404:0x208:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826407.523320:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826407.523340:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1787826407.523347:0:69451:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x208:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826407.523353:0:69451:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) 00000080:00010000:2.0:1787826407.523357:0:69451:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) for lock 0xe5f736940357bafe 00000080:00010000:2.0:1787826407.523362:0:69451:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357bafe from it ffff9dd142bd7cd8 00010000:00010000:2.0:1787826407.523365:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1787826407.523374:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1787826407.523397:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/1, layout version 6 restore needed 0 00020000:00200000:2.0:1787826407.523408:0:69451:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826407.523412:0:69451:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1787826407.523416:0:69451:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x209:0x0] 00000008:00010000:2.0:1787826407.523433:0:69451:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826407.523437:0:69451:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d963222b0, osc lock ffff907d830a29c0, flags 0x400000400001002 00010000:00010000:2.0:1787826407.523454:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1088a expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826407.523465:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 104857599) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1088a expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1787826407.523477:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826407.523481:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc1088a expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826407.523489:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc1088a expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1787826407.523499:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826407.523509:0:69451:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x209:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826407.523514:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_release():245 00000020:00010000:2.0:1787826407.523517:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826407.523521:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d963222b0 (ffff907da5608d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826407.523524:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826407.523527:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d963222b0 (ffff907d884a7908/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826407.523548:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/1, layout version 6 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:2.0:1787826407.523585:0:69451:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8) cmd=c00c66f3 arg=15f02a0 00000002:00010000:2.0:1787826407.523601:0:69451:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x209:0x0]) in obj [0x200000404:0x209:0x0], intent: open flags 05004040000002 00010000:00010000:2.0:1787826407.523671:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ac00/0xe5f736940357bb13 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 69451 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.523679:0:69451:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x81000000001000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ac00/0xe5f736940357bb13 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x0 pid: 69451 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826407.523689:0:69451:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826407.523697:0:69451:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114bc00 x1874670011471232/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826407.523706:0:69451:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ac00/0xe5f736940357bb13 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x81000000000000 pid: 69451 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826407.523725:0:69451:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011471232:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826407.523915:0:69451:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d87248900 going to sleep for 13 seconds 00000100:00100000:2.0:1787826407.532861:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011470976:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1787826407.536468:0:69451:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011471232:192.168.203.104@tcp:101:lfs.0 00010000:00010000:2.0:1787826407.536491:0:69451:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PW ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ac00/0xe5f736940357bb13 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc108ad expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.536503:0:69451:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ac00/0xe5f736940357bb13 lrc: 4/1,0 mode: --/PW res: [0x200000404:0x209:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc108ad expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.536516:0:69451:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ac00/0xe5f736940357bb13 lrc: 4/1,0 mode: PW/PW res: [0x200000404:0x209:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc108ad expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.536527:0:69451:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ac00/0xe5f736940357bb13 lrc: 4/1,0 mode: PW/PW res: [0x200000404:0x209:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc108ad expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.536549:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ac00/0xe5f736940357bb13 lrc: 5/1,1 mode: PW/PW res: [0x200000404:0x209:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc108ad expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.536557:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ac00/0xe5f736940357bb13 lrc: 5/1,1 mode: PW/PW res: [0x200000404:0x209:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc108ad expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.536565:0:69451:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ac00/0xe5f736940357bb13 lrc: 4/0,1 mode: PW/PW res: [0x200000404:0x209:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc108ad expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1787826407.536576:0:69451:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x600002b, status=0 req@ffff907d9114bc00 x1874670011471232/t8589943035(8589943035) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1160 e 0 to 0 dl 1787826425 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1787826407.536589:0:69451:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d9114bc00 x1874670011471232/t8589943035(8589943035) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/1160 e 0 to 0 dl 1787826425 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1787826407.536604:0:69451:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ac00/0xe5f736940357bb13 lrc: 3/0,1 mode: PW/PW res: [0x200000404:0x209:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7918213f4fc108ad expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.536650:0:69451:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 2 res 8589935620/521 (0 0) 00000080:00200000:2.0:1787826407.536656:0:69451:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x209:0x0] suppgids 0 0: rc 0 00000002:00100000:2.0:1787826407.536671:0:69451:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff907d9114bc00 x1874670011471232/t8589943035(8589943035) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826425 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1787826407.536680:0:69451:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) for lock 0xe5f736940357bb13 00000080:00010000:2.0:1787826407.536685:0:69451:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357bb13 from it ffff9dd142bd7cc0 00010000:00010000:2.0:1787826407.536689:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ac00/0xe5f736940357bb13 lrc: 3/0,1 mode: PW/PW res: [0x200000404:0x209:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc108ad expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.536697:0:69451:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ac00/0xe5f736940357bb13 lrc: 2/0,0 mode: PW/PW res: [0x200000404:0x209:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7918213f4fc108ad expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1787826407.536749:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 6 restore needed 0 00020000:00200000:2.0:1787826407.536760:0:69451:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826407.536763:0:69451:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1787826407.536767:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1787826407.536776:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000100:00100000:2.0:1787826407.536836:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1787826407.536853:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011471360:192.168.203.104@tcp:1:lfs.0 00000100:00100000:3.0:1787826407.550128:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826407.550903:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025522304 00000100:00100000:2.0:1787826407.550913:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025522304 00000100:00100000:2.0:1787826407.550977:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7165 00000100:00100000:2.0:1787826407.550987:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d835270c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:36729:x1874670025522304:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826407.551005:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc1088a expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826407.551142:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826407.551150:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d835270c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:36729:x1874670025522304:12345-192.168.203.104@tcp:104: Request processed in 164us (1031us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826407.551158:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7165 00010000:00010000:0.0:1787826407.553650:0:40347:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826407.553664:0:40347:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc1088a expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826407.553677:0:40347:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0b400 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826407.553684:0:40347:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800428400000000 nid: local remote: 0x7918213f4fc1088a expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826407.553712:0:40347:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1787826407.555522:0:40347:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1787826407.556474:0:40347:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 2/0,0 mode: --/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc1088a expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826407.556488:0:40347:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826407.556530:0:40347:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc1088a expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826407.556548:0:40347:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000060d9cb76/0xe5f736940357bb05 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc1088a expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826407.556575:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826407.556577:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826407.556580:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826407.556602:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1787826407.556752:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011471488:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826407.560042:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011471488:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1787826407.560175:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011471360:192.168.203.104@tcp:1:lfs.0 00010000:00010000:0.0:1787826407.560359:0:69451:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1787826407.560382:0:69451:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x209:0x0]: losing layout lock 00010000:00010000:0.0:1787826407.560401:0:69451:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1787826407.560414:0:69451:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x209:0x0]: losing layout lock 00010000:00010000:0.0:1787826407.560444:0:69451:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 1/0,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1787826407.560452:0:69451:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 1/0,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826407.560457:0:69451:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1787826407.560460:0:69451:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bafe lrc: 0/0,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc10883 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1787826407.560473:0:69451:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0bc00/0xe5f736940357babf lrc: 0/0,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc1075d expref: -99 pid: 69408 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000020:00100000:0.0:1787826407.560488:0:69451:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1787826407.560496:0:69451:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90845a40 x1874670011471744/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 472/440 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787826407.560518:0:69451:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011471744:192.168.203.104@tcp:36:lfs.0 00000100:00100000:0.0:1787826407.560603:0:69451:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238600 going to sleep for 13 seconds 00000100:00100000:0.0:1787826407.570119:0:69451:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589943036 gen 1 00000100:00100000:0.0:1787826407.570142:0:69451:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d908a03c0 x1874670011469056/t8589943033(8589943033) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826425 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1787826407.570156:0:69451:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d9114a580 x1874670011470848/t8589943034(8589943034) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826425 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1787826407.570168:0:69451:0:(client.c:3174:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff907d9114bc00 x1874670011471232/t8589943035(8589943035) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826425 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1787826407.570182:0:69451:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011471744:192.168.203.104@tcp:36:lfs.0 00010000:00010000:0.0:1787826407.570233:0:69451:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 31 res 8589935620/521 (0 0) 00000080:00010000:0.0:1787826407.570247:0:69451:0:(file.c:7076:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000404:0x209:0x0](ffff907d933121c8) 00000002:00010000:0.0:1787826407.570259:0:69451:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x209:0x0]) in obj [0x200000404:0x209:0x0], intent: layout flags 0 00010000:00010000:0.0:1787826407.570311:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 3/1,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 69451 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.570321:0:69451:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 3/1,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69451 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.570331:0:69451:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 3/1,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69451 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826407.570350:0:69451:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011471872:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826407.570444:0:69451:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238600 going to sleep for 13 seconds 00000100:00100000:0.0:1787826407.584168:0:69451:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011471872:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826407.584214:0:69451:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.584231:0:69451:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826407.584250:0:69451:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x400 disposition=0x1, status=0 req@ffff907d90845a40 x1874670011471872/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/896 e 0 to 0 dl 1787826425 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826407.584263:0:69451:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1787826407.584286:0:69451:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.584302:0:69451:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/521 (0 0) 00000080:00200000:0.0:1787826407.584310:0:69451:0:(namei.c:952:ll_intent_lock()) intent lock 1024 on i1 [0x200000404:0x209:0x0] suppgids 0 0: rc 0 00000080:00010000:0.0:1787826407.584344:0:69451:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) for lock 0xe5f736940357bb1a 00000080:00010000:0.0:1787826407.584354:0:69451:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357bb1a from it ffff9dd142bd7c70 00010000:00010000:0.0:1787826407.584359:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.584371:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826407.584409:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826407.584424:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:0.0:1787826407.584434:0:69451:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x209:0x0](000000003c30eba3) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826407.584488:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826407.584497:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826407.584514:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826407.584527:0:69451:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826407.584531:0:69451:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826407.584549:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826407.584562:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:0.0:1787826407.584587:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1787826407.584602:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826407.584661:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:0.0:1787826407.584669:0:69451:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x209:0x0](000000003c30eba3) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:0.0:1787826407.584688:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1787826407.584694:0:69451:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826407.584696:0:69451:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:0.0:1787826407.584699:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1787826407.584704:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:0.0:1787826407.584710:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1787826407.584805:0:69451:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x209:0x0]: layout version change: 4294967294 -> 7 00010000:00010000:0.0:1787826407.584811:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826407.584819:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200080:0.0:1787826407.584853:0:69451:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8) cmd=66f4 arg=8000000 00000080:00200008:0.0:1787826407.584884:0:69451:0:(file.c:6313:ll_getattr_dentry()) START file f204e.sanity-flr:[0x200000404:0x209:0x0](ffff907d933121c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1787826407.584893:0:69451:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8),name=f204e.sanity-flr 00000002:00010000:0.0:1787826407.584913:0:69451:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x209:0x0]) in obj [0x200000404:0x209:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826407.584921:0:69451:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 30 res 8589935620/521 (0 0) 00010000:00010000:0.0:1787826407.584969:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 69451 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.584977:0:69451:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 69451 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.584987:0:69451:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 69451 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826407.585014:0:69451:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011472000:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826407.585137:0:69451:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238600 going to sleep for 13 seconds 00000100:00100000:0.0:1787826407.592651:0:69451:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011472000:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826407.592693:0:69451:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.592709:0:69451:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.592729:0:69451:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.592750:0:69451:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.592767:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.592777:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.592787:0:69451:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826407.592800:0:69451:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d90845a40 x1874670011472000/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 576/1136 e 0 to 0 dl 1787826425 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1787826407.592813:0:69451:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:0.0:1787826407.592832:0:69451:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.592847:0:69451:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/521 (0 0) 00000080:00200000:0.0:1787826407.592856:0:69451:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x209:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1787826407.592965:0:69451:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x209:0x0]: layout version change: 7 -> 7 00000080:00010000:0.0:1787826407.592972:0:69451:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) for lock 0xe5f736940357bb21 00000080:00010000:0.0:1787826407.592979:0:69451:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) 00000080:00010000:0.0:1787826407.592983:0:69451:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357bb21 from it ffff9dd142bd7cd8 00010000:00010000:0.0:1787826407.592987:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.592998:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826407.593048:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/1, layout version 6 restore needed 0 00020000:00200000:0.0:1787826407.593062:0:69451:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826407.593066:0:69451:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:0.0:1787826407.593071:0:69451:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x209:0x0] 00000008:00010000:0.0:1787826407.593093:0:69451:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1787826407.593097:0:69451:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da032e9b0, osc lock ffff907d96368000, flags 0x400000400001002 00010000:00010000:0.0:1787826407.593162:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.593174:0:69451:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 3/1,0 mode: --/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->104857599] (req 0->104857599) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.593184:0:69451:0:(ldlm_request.c:1139:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff907d90845a40 x1874670011472128/t0(0) o101->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000020:00200000:0.0:1787826407.593223:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:1.0:1787826407.593319:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011472128:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826407.597820:0:2348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->104857599] (req 0->104857599) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.597857:0:2348:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 6/2,0 mode: --/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.597887:0:2348:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.597901:0:2348:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826407.597921:0:2348:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 5/2,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00200000:1.0:1787826407.597935:0:2348:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1787826407.597962:0:2348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.597972:0:2348:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.597980:0:2348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.597991:0:2348:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00100000:1.0:1787826407.598018:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011472128:192.168.203.104@tcp:101:lfs.0 00000080:00200000:0.0:1787826407.598137:0:69451:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x209:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:0.0:1787826407.598145:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_release():245 00000020:00010000:0.0:1787826407.598150:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1787826407.598156:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da032e9b0 (ffff907da5608d88/1) at cl_lock_cancel():142 00000020:00010000:0.0:1787826407.598161:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5790278 (ffff907d884a7908/0) at cl_lock_fini():72 00000020:00010000:0.0:1787826407.598164:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da032e9b0 (ffff907d884a7908/1) at cl_lock_fini():72 00000080:00200000:0.0:1787826407.598183:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:2.0:1787826407.663216:0:69451:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8) cmd=400866b1 arg=1 00000080:00200008:2.0:1787826407.663238:0:69451:0:(file.c:6313:ll_getattr_dentry()) START file f204e.sanity-flr:[0x200000404:0x209:0x0](ffff907d933121c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1787826407.663248:0:69451:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8),name=f204e.sanity-flr 00000002:00010000:2.0:1787826407.663271:0:69451:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x209:0x0]) in obj [0x200000404:0x209:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826407.663293:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826407.663310:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826407.663318:0:69451:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x209:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826407.663328:0:69451:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) 00000080:00010000:2.0:1787826407.663332:0:69451:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) for lock 0xe5f736940357bb21 00000080:00010000:2.0:1787826407.663340:0:69451:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357bb21 from it ffff9dd142bd7cd8 00010000:00010000:2.0:1787826407.663345:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826407.663354:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1787826407.663383:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/1, layout version 6 restore needed 0 00020000:00200000:2.0:1787826407.663395:0:69451:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826407.663398:0:69451:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1787826407.663402:0:69451:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x209:0x0] 00000008:00010000:2.0:1787826407.663420:0:69451:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826407.663423:0:69451:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d963222b0, osc lock ffff907d830a29c0, flags 0x400000400001002 00010000:00010000:2.0:1787826407.663438:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826407.663449:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 104857599) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:2.0:1787826407.665641:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826407.665649:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826407.665663:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:2.0:1787826407.665680:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1787826407.665859:0:69451:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x209:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:3.0:1787826407.665865:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5792c70 (ffff907d884a7d88/0) at cl_lock_release():245 00000020:00010000:3.0:1787826407.665868:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5792c70 (ffff907d884a7d88/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826407.665872:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d963222b0 (ffff907d884a71b8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826407.665876:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5792c70 (ffff907d884a7d88/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826407.665879:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d963222b0 (ffff907d884a7d88/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826407.665893:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:3.0:1787826407.665920:0:69451:0:(file.c:5224:ll_file_seek()) START file f204e.sanity-flr:[0x200000404:0x209:0x0], offset: 0, type: SEEK_DATA 00000080:00200000:3.0:1787826407.665926:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:3.0:1787826407.665934:0:69451:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:3.0:1787826407.665937:0:69451:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:3.0:1787826407.665942:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1787826407.665949:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:3.0:1787826407.665963:0:69451:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x209:0x0] updating i_size 2097152 i_blocks 4096 00000080:00200000:3.0:1787826407.665970:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787826407.665975:0:69451:0:(file.c:1848:ll_io_set_mirror()) f204e.sanity-flr: desiginated mirror: 1 00000080:00200000:3.0:1787826407.665978:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:3.0:1787826407.665983:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1787826407.665986:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:3.0:1787826407.665989:0:69451:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 18446744073709551615] 00000008:00010000:3.0:1787826407.666001:0:69451:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:3.0:1787826407.666004:0:69451:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d98a1ceb0, osc lock ffff907d963b9c08, flags 0x0 00010000:00010000:3.0:1787826407.666019:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826407.666032:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 104857599) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826407.666042:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826407.666050:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826407.666057:0:69451:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787826407.666066:0:69451:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x209:0x0] updating i_size 2097152 i_blocks 4096 00000100:00100000:3.0:1787826407.666129:0:2351:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1787826407.666148:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011472256:192.168.203.104@tcp:23:lfs.0 00000100:00100000:2.0:1787826407.678863:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011472256:192.168.203.104@tcp:23:lfs.0 00000020:00010000:3.0:1787826407.682177:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_release():245 00000020:00010000:3.0:1787826407.682189:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826407.682194:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98a1ceb0 (ffff907d884a71b8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787826407.682205:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826407.682222:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00010000:3.0:1787826407.682239:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826407.682242:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98a1ceb0 (ffff907d884a7d88/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826407.682263:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:3.0:1787826407.682271:0:69451:0:(file.c:5262:ll_file_seek()) COMPLETED file f204e.sanity-flr:[0x200000404:0x209:0x0], offset: 0, type: SEEK_DATA, rc = 0 00000080:00200008:3.0:1787826407.682283:0:69451:0:(file.c:5224:ll_file_seek()) START file f204e.sanity-flr:[0x200000404:0x209:0x0], offset: 0, type: SEEK_HOLE 00000080:00200000:3.0:1787826407.682292:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:3.0:1787826407.682300:0:69451:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:3.0:1787826407.682302:0:69451:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:3.0:1787826407.682307:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1787826407.682314:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:3.0:1787826407.682344:0:69451:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x209:0x0] updating i_size 2097152 i_blocks 4096 00000080:00200000:3.0:1787826407.682351:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1787826407.682357:0:69451:0:(file.c:1848:ll_io_set_mirror()) f204e.sanity-flr: desiginated mirror: 1 00000080:00200000:3.0:1787826407.682359:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:3.0:1787826407.682365:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:3.0:1787826407.682368:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:3.0:1787826407.682370:0:69451:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 18446744073709551615] 00000008:00010000:3.0:1787826407.682386:0:69451:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:3.0:1787826407.682390:0:69451:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d98a1ceb0, osc lock ffff907d963b9c08, flags 0x0 00010000:00010000:3.0:1787826407.682409:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826407.682424:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 104857599) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826407.682436:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826407.682444:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826407.682451:0:69451:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1787826407.682472:0:69451:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x209:0x0] updating i_size 2097152 i_blocks 4096 00000100:00100000:2.0:1787826407.682580:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1787826407.682603:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011472384:192.168.203.104@tcp:23:lfs.0 00000100:00100000:2.0:1787826407.695659:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011472384:192.168.203.104@tcp:23:lfs.0 00000020:00010000:3.0:1787826407.695818:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_release():245 00000020:00010000:3.0:1787826407.695823:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826407.695828:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98a1ceb0 (ffff907d884a71b8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1787826407.695838:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1787826407.695855:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:3.0:1787826407.695870:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826407.695873:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98a1ceb0 (ffff907d884a7d88/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826407.695890:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:3.0:1787826407.695963:0:69451:0:(file.c:5262:ll_file_seek()) COMPLETED file f204e.sanity-flr:[0x200000404:0x209:0x0], offset: 0, type: SEEK_HOLE, rc = 2097152 00000080:00200080:3.0:1787826407.699273:0:69451:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8) cmd=400866b1 arg=0 00000080:00200080:3.0:1787826407.699298:0:69451:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8) cmd=400866b1 arg=1 00000080:00200008:3.0:1787826407.699305:0:69451:0:(file.c:6313:ll_getattr_dentry()) START file f204e.sanity-flr:[0x200000404:0x209:0x0](ffff907d933121c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787826407.699313:0:69451:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8),name=f204e.sanity-flr 00000002:00010000:3.0:1787826407.699336:0:69451:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x209:0x0]) in obj [0x200000404:0x209:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826407.699354:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826407.699369:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1787826407.699378:0:69451:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x209:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826407.700135:0:69451:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) 00000080:00010000:3.0:1787826407.700141:0:69451:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) for lock 0xe5f736940357bb21 00000080:00010000:3.0:1787826407.700150:0:69451:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357bb21 from it ffff9dd142bd7cd8 00010000:00010000:3.0:1787826407.700154:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1787826407.700167:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1787826407.700195:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:3.0:1787826407.700207:0:69451:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:3.0:1787826407.700210:0:69451:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:3.0:1787826407.700214:0:69451:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x209:0x0] 00000008:00010000:3.0:1787826407.700233:0:69451:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826407.700236:0:69451:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d98a1ceb0, osc lock ffff907d963b9c08, flags 0x400000400001002 00010000:00010000:3.0:1787826407.700253:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1787826407.700271:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 104857599) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:3.0:1787826407.700284:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826407.700288:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1787826407.700296:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:3.0:1787826407.700306:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1787826407.700315:0:69451:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x209:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:3.0:1787826407.700319:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5792c70 (ffff907d884a7d88/0) at cl_lock_release():245 00000020:00010000:3.0:1787826407.700322:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5792c70 (ffff907d884a7d88/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826407.700324:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98a1ceb0 (ffff907d884a71b8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826407.700328:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5792c70 (ffff907d884a7d88/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826407.700331:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98a1ceb0 (ffff907d884a7d88/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826407.700344:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:3.0:1787826407.700370:0:69451:0:(file.c:2485:do_file_read_iter()) START file f204e.sanity-flr:[0x200000404:0x209:0x0], ppos: 0, count: 2097152 00000080:00200000:3.0:1787826407.700379:0:69451:0:(file.c:2090:ll_file_io_generic()) f204e.sanity-flr: read ppos: 0, bytes: 2097152 00000080:00200000:3.0:1787826407.700387:0:69451:0:(file.c:1848:ll_io_set_mirror()) f204e.sanity-flr: desiginated mirror: 1 00000020:00200000:3.0:1787826407.700454:0:69451:0:(cl_io.c:218:cl_io_rw_init()) header@0000000033ed2c48[0x110, 2, [0x200000404:0x209:0x0] hash] 00000020:00200000:3.0:1787826407.700797:0:69451:0:(cl_io.c:218:cl_io_rw_init()) io range: 1 [0, 2097152) 2048 0 00000080:00200000:3.0:1787826407.700801:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00000080:00200000:3.0:1787826407.700829:0:69451:0:(file.c:2170:ll_file_io_generic()) Range lock [0, 511] 00020000:00200000:3.0:1787826407.700837:0:69451:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 2097152) 2097152, 2097152 00020000:00200000:3.0:1787826407.700841:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1787826407.700859:0:69451:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 511] 00000008:00010000:3.0:1787826407.700869:0:69451:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 11 00000008:00010000:3.0:1787826407.700872:0:69451:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d98a1ceb0, osc lock ffff907d963b9c08, flags 0x40000 00000080:00200000:3.0:1787826407.700878:0:69451:0:(vvp_io.c:874:vvp_io_read_start()) f204e.sanity-flr: read [0, 2097152) 00000080:00200000:3.0:1787826407.700883:0:69451:0:(vvp_io.c:194:vvp_prep_size()) [0x200000404:0x209:0x0] updating i_size 2097152 00000080:00200000:3.0:1787826407.700891:0:69451:0:(vvp_io.c:932:vvp_io_read_start()) count:2097152 iocb pos:0 00000080:00200000:3.0:1787826407.700897:0:69451:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8), size=2097152 (max 335544320), offset=0=0x0, pages 512 (max 81920), lockless, parallel, hybrid 00000020:00200000:3.0:1787826407.701335:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 512 items remaining. 00000080:00200000:3.0:1787826407.701338:0:69451:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 2097152 count 2097152 file_offset 2097152 00000020:00010000:3.0:1787826407.701345:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_release():245 00000020:00010000:3.0:1787826407.701347:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826407.701350:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98a1ceb0 (ffff907d884a71b8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826407.701352:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826407.701354:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98a1ceb0 (ffff907d884a7d88/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826407.701361:0:69451:0:(vvp_io.c:533:vvp_io_advance()) advancing 2097152 bytes 00000020:00200000:3.0:1787826407.701366:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:3.0:1787826407.701381:0:61776:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907d83b79540 00000008:00100000:3.0:1787826407.702102:0:61776:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d83527840 - object 0x240000400:809 offset 0<>2097152 00000100:00100000:2.0:1787826407.702164:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011472512:192.168.203.104@tcp:3:lfs.0 00000008:00010000:2.0:1787826407.792331:0:2350:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907d83b79540 00000020:00200000:2.0:1787826407.792342:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 511 items remaining. 00000020:00200000:2.0:1787826407.792347:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 510 items remaining. 00000020:00200000:2.0:1787826407.792348:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 509 items remaining. 00000020:00200000:2.0:1787826407.792350:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 508 items remaining. 00000020:00200000:2.0:1787826407.792351:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 507 items remaining. 00000020:00200000:2.0:1787826407.792353:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 506 items remaining. 00000020:00200000:2.0:1787826407.792354:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 505 items remaining. 00000020:00200000:2.0:1787826407.792356:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 504 items remaining. 00000020:00200000:2.0:1787826407.792357:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 503 items remaining. 00000020:00200000:2.0:1787826407.792359:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 502 items remaining. 00000020:00200000:2.0:1787826407.792361:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 501 items remaining. 00000020:00200000:2.0:1787826407.792363:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 500 items remaining. 00000020:00200000:2.0:1787826407.792365:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 499 items remaining. 00000020:00200000:2.0:1787826407.792367:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 498 items remaining. 00000020:00200000:2.0:1787826407.792369:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 497 items remaining. 00000020:00200000:2.0:1787826407.792370:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 496 items remaining. 00000020:00200000:2.0:1787826407.792372:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 495 items remaining. 00000020:00200000:2.0:1787826407.792373:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 494 items remaining. 00000020:00200000:2.0:1787826407.792375:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 493 items remaining. 00000020:00200000:2.0:1787826407.792376:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 492 items remaining. 00000020:00200000:2.0:1787826407.792378:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 491 items remaining. 00000020:00200000:2.0:1787826407.792379:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 490 items remaining. 00000020:00200000:2.0:1787826407.792381:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 489 items remaining. 00000020:00200000:2.0:1787826407.792383:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 488 items remaining. 00000020:00200000:2.0:1787826407.792384:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 487 items remaining. 00000020:00200000:2.0:1787826407.792386:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 486 items remaining. 00000020:00200000:2.0:1787826407.792388:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 485 items remaining. 00000020:00200000:2.0:1787826407.792901:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 484 items remaining. 00000020:00200000:2.0:1787826407.792904:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 483 items remaining. 00000020:00200000:2.0:1787826407.792906:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 482 items remaining. 00000020:00200000:2.0:1787826407.792908:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 481 items remaining. 00000020:00200000:2.0:1787826407.792910:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 480 items remaining. 00000020:00200000:2.0:1787826407.792911:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 479 items remaining. 00000020:00200000:2.0:1787826407.792913:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 478 items remaining. 00000020:00200000:2.0:1787826407.792915:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 477 items remaining. 00000020:00200000:2.0:1787826407.792917:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 476 items remaining. 00000020:00200000:2.0:1787826407.792918:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 475 items remaining. 00000020:00200000:2.0:1787826407.792920:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 474 items remaining. 00000020:00200000:2.0:1787826407.792921:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 473 items remaining. 00000020:00200000:2.0:1787826407.792922:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 472 items remaining. 00000020:00200000:2.0:1787826407.792924:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 471 items remaining. 00000020:00200000:2.0:1787826407.792925:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 470 items remaining. 00000020:00200000:2.0:1787826407.792927:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 469 items remaining. 00000020:00200000:2.0:1787826407.792928:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 468 items remaining. 00000020:00200000:2.0:1787826407.792930:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 467 items remaining. 00000020:00200000:2.0:1787826407.792931:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 466 items remaining. 00000020:00200000:2.0:1787826407.792933:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 465 items remaining. 00000020:00200000:2.0:1787826407.792935:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 464 items remaining. 00000020:00200000:2.0:1787826407.792936:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 463 items remaining. 00000020:00200000:2.0:1787826407.792937:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 462 items remaining. 00000020:00200000:2.0:1787826407.792939:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 461 items remaining. 00000020:00200000:2.0:1787826407.792940:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 460 items remaining. 00000020:00200000:2.0:1787826407.792942:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 459 items remaining. 00000020:00200000:2.0:1787826407.792943:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 458 items remaining. 00000020:00200000:2.0:1787826407.792945:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 457 items remaining. 00000020:00200000:2.0:1787826407.792946:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 456 items remaining. 00000020:00200000:2.0:1787826407.792948:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 455 items remaining. 00000020:00200000:2.0:1787826407.792949:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 454 items remaining. 00000020:00200000:2.0:1787826407.792950:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 453 items remaining. 00000020:00200000:2.0:1787826407.792952:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 452 items remaining. 00000020:00200000:2.0:1787826407.792953:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 451 items remaining. 00000020:00200000:2.0:1787826407.792954:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 450 items remaining. 00000020:00200000:2.0:1787826407.792956:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 449 items remaining. 00000020:00200000:2.0:1787826407.792957:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 448 items remaining. 00000020:00200000:2.0:1787826407.792959:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 447 items remaining. 00000020:00200000:2.0:1787826407.792960:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 446 items remaining. 00000020:00200000:2.0:1787826407.792961:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 445 items remaining. 00000020:00200000:2.0:1787826407.792963:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 444 items remaining. 00000020:00200000:2.0:1787826407.792965:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 443 items remaining. 00000020:00200000:2.0:1787826407.792966:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 442 items remaining. 00000020:00200000:2.0:1787826407.792968:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 441 items remaining. 00000020:00200000:2.0:1787826407.792970:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 440 items remaining. 00000020:00200000:2.0:1787826407.792971:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 439 items remaining. 00000020:00200000:2.0:1787826407.792973:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 438 items remaining. 00000020:00200000:2.0:1787826407.792975:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 437 items remaining. 00000020:00200000:2.0:1787826407.792977:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 436 items remaining. 00000020:00200000:2.0:1787826407.792978:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 435 items remaining. 00000020:00200000:2.0:1787826407.792980:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 434 items remaining. 00000020:00200000:2.0:1787826407.792982:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 433 items remaining. 00000020:00200000:2.0:1787826407.792984:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 432 items remaining. 00000020:00200000:2.0:1787826407.792987:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 431 items remaining. 00000020:00200000:2.0:1787826407.792988:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 430 items remaining. 00000020:00200000:2.0:1787826407.792990:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 429 items remaining. 00000020:00200000:2.0:1787826407.792992:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 428 items remaining. 00000020:00200000:2.0:1787826407.792994:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 427 items remaining. 00000020:00200000:2.0:1787826407.792995:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 426 items remaining. 00000020:00200000:2.0:1787826407.792997:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 425 items remaining. 00000020:00200000:2.0:1787826407.792999:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 424 items remaining. 00000020:00200000:2.0:1787826407.793000:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 423 items remaining. 00000020:00200000:2.0:1787826407.793002:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 422 items remaining. 00000020:00200000:2.0:1787826407.793004:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 421 items remaining. 00000020:00200000:2.0:1787826407.793006:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 420 items remaining. 00000020:00200000:2.0:1787826407.793007:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 419 items remaining. 00000020:00200000:2.0:1787826407.793009:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 418 items remaining. 00000020:00200000:2.0:1787826407.793011:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 417 items remaining. 00000020:00200000:2.0:1787826407.793013:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 416 items remaining. 00000020:00200000:2.0:1787826407.793014:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 415 items remaining. 00000020:00200000:2.0:1787826407.793016:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 414 items remaining. 00000020:00200000:2.0:1787826407.793018:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 413 items remaining. 00000020:00200000:2.0:1787826407.793019:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 412 items remaining. 00000020:00200000:2.0:1787826407.793021:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 411 items remaining. 00000020:00200000:2.0:1787826407.793023:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 410 items remaining. 00000020:00200000:2.0:1787826407.793025:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 409 items remaining. 00000020:00200000:2.0:1787826407.793026:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 408 items remaining. 00000020:00200000:2.0:1787826407.793028:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 407 items remaining. 00000020:00200000:2.0:1787826407.793029:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 406 items remaining. 00000020:00200000:2.0:1787826407.793031:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 405 items remaining. 00000020:00200000:2.0:1787826407.793033:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 404 items remaining. 00000020:00200000:2.0:1787826407.793035:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 403 items remaining. 00000020:00200000:2.0:1787826407.793036:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 402 items remaining. 00000020:00200000:2.0:1787826407.793038:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 401 items remaining. 00000020:00200000:2.0:1787826407.793040:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 400 items remaining. 00000020:00200000:2.0:1787826407.793042:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 399 items remaining. 00000020:00200000:2.0:1787826407.793044:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 398 items remaining. 00000020:00200000:2.0:1787826407.793045:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 397 items remaining. 00000020:00200000:2.0:1787826407.793047:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 396 items remaining. 00000020:00200000:2.0:1787826407.793049:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 395 items remaining. 00000020:00200000:2.0:1787826407.793050:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 394 items remaining. 00000020:00200000:2.0:1787826407.793052:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 393 items remaining. 00000020:00200000:2.0:1787826407.793053:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 392 items remaining. 00000020:00200000:2.0:1787826407.793055:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 391 items remaining. 00000020:00200000:2.0:1787826407.793056:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 390 items remaining. 00000020:00200000:2.0:1787826407.793058:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 389 items remaining. 00000020:00200000:2.0:1787826407.793060:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 388 items remaining. 00000020:00200000:2.0:1787826407.793062:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 387 items remaining. 00000020:00200000:2.0:1787826407.793063:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 386 items remaining. 00000020:00200000:2.0:1787826407.793065:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 385 items remaining. 00000020:00200000:2.0:1787826407.793066:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 384 items remaining. 00000020:00200000:2.0:1787826407.793068:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 383 items remaining. 00000020:00200000:2.0:1787826407.793070:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 382 items remaining. 00000020:00200000:2.0:1787826407.793072:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 381 items remaining. 00000020:00200000:2.0:1787826407.793073:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 380 items remaining. 00000020:00200000:2.0:1787826407.793075:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 379 items remaining. 00000020:00200000:2.0:1787826407.793076:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 378 items remaining. 00000020:00200000:2.0:1787826407.793078:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 377 items remaining. 00000020:00200000:2.0:1787826407.793079:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 376 items remaining. 00000020:00200000:2.0:1787826407.793081:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 375 items remaining. 00000020:00200000:2.0:1787826407.793082:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 374 items remaining. 00000020:00200000:2.0:1787826407.793084:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 373 items remaining. 00000020:00200000:2.0:1787826407.793086:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 372 items remaining. 00000020:00200000:2.0:1787826407.793088:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 371 items remaining. 00000020:00200000:2.0:1787826407.793090:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 370 items remaining. 00000020:00200000:2.0:1787826407.793092:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 369 items remaining. 00000020:00200000:2.0:1787826407.793093:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 368 items remaining. 00000020:00200000:2.0:1787826407.793095:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 367 items remaining. 00000020:00200000:2.0:1787826407.793097:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 366 items remaining. 00000020:00200000:2.0:1787826407.793099:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 365 items remaining. 00000020:00200000:2.0:1787826407.793101:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 364 items remaining. 00000020:00200000:2.0:1787826407.793102:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 363 items remaining. 00000020:00200000:2.0:1787826407.793104:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 362 items remaining. 00000020:00200000:2.0:1787826407.793106:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 361 items remaining. 00000020:00200000:2.0:1787826407.793107:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 360 items remaining. 00000020:00200000:2.0:1787826407.793109:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 359 items remaining. 00000020:00200000:2.0:1787826407.793111:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 358 items remaining. 00000020:00200000:2.0:1787826407.793113:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 357 items remaining. 00000020:00200000:2.0:1787826407.793114:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 356 items remaining. 00000020:00200000:2.0:1787826407.793116:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 355 items remaining. 00000020:00200000:2.0:1787826407.793117:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 354 items remaining. 00000020:00200000:2.0:1787826407.793119:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 353 items remaining. 00000020:00200000:2.0:1787826407.793121:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 352 items remaining. 00000020:00200000:2.0:1787826407.793123:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 351 items remaining. 00000020:00200000:2.0:1787826407.793125:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 350 items remaining. 00000020:00200000:2.0:1787826407.793127:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 349 items remaining. 00000020:00200000:2.0:1787826407.793129:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 348 items remaining. 00000020:00200000:2.0:1787826407.793131:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 347 items remaining. 00000020:00200000:2.0:1787826407.793132:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 346 items remaining. 00000020:00200000:2.0:1787826407.793134:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 345 items remaining. 00000020:00200000:2.0:1787826407.793136:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 344 items remaining. 00000020:00200000:2.0:1787826407.793138:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 343 items remaining. 00000020:00200000:2.0:1787826407.793139:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 342 items remaining. 00000020:00200000:2.0:1787826407.793141:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 341 items remaining. 00000020:00200000:2.0:1787826407.793143:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 340 items remaining. 00000020:00200000:2.0:1787826407.793145:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 339 items remaining. 00000020:00200000:2.0:1787826407.793147:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 338 items remaining. 00000020:00200000:2.0:1787826407.793149:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 337 items remaining. 00000020:00200000:2.0:1787826407.793151:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 336 items remaining. 00000020:00200000:2.0:1787826407.793152:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 335 items remaining. 00000020:00200000:2.0:1787826407.793154:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 334 items remaining. 00000020:00200000:2.0:1787826407.793155:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 333 items remaining. 00000020:00200000:2.0:1787826407.793157:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 332 items remaining. 00000020:00200000:2.0:1787826407.793159:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 331 items remaining. 00000020:00200000:2.0:1787826407.793160:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 330 items remaining. 00000020:00200000:2.0:1787826407.793162:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 329 items remaining. 00000020:00200000:2.0:1787826407.793164:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 328 items remaining. 00000020:00200000:2.0:1787826407.793165:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 327 items remaining. 00000020:00200000:2.0:1787826407.793167:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 326 items remaining. 00000020:00200000:2.0:1787826407.793169:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 325 items remaining. 00000020:00200000:2.0:1787826407.793170:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 324 items remaining. 00000020:00200000:2.0:1787826407.793172:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 323 items remaining. 00000020:00200000:2.0:1787826407.793173:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 322 items remaining. 00000020:00200000:2.0:1787826407.793175:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 321 items remaining. 00000020:00200000:2.0:1787826407.793176:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 320 items remaining. 00000020:00200000:2.0:1787826407.793178:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 319 items remaining. 00000020:00200000:2.0:1787826407.793180:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 318 items remaining. 00000020:00200000:2.0:1787826407.793182:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 317 items remaining. 00000020:00200000:2.0:1787826407.793184:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 316 items remaining. 00000020:00200000:2.0:1787826407.793185:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 315 items remaining. 00000020:00200000:2.0:1787826407.793187:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 314 items remaining. 00000020:00200000:2.0:1787826407.793189:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 313 items remaining. 00000020:00200000:2.0:1787826407.793191:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 312 items remaining. 00000020:00200000:2.0:1787826407.793193:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 311 items remaining. 00000020:00200000:2.0:1787826407.793195:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 310 items remaining. 00000020:00200000:2.0:1787826407.793196:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 309 items remaining. 00000020:00200000:2.0:1787826407.793199:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 308 items remaining. 00000020:00200000:2.0:1787826407.793200:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 307 items remaining. 00000020:00200000:2.0:1787826407.793202:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 306 items remaining. 00000020:00200000:2.0:1787826407.793204:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 305 items remaining. 00000020:00200000:2.0:1787826407.793205:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 304 items remaining. 00000020:00200000:2.0:1787826407.793207:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 303 items remaining. 00000020:00200000:2.0:1787826407.793209:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 302 items remaining. 00000020:00200000:2.0:1787826407.793210:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 301 items remaining. 00000020:00200000:2.0:1787826407.793212:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 300 items remaining. 00000020:00200000:2.0:1787826407.793213:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 299 items remaining. 00000020:00200000:2.0:1787826407.793215:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 298 items remaining. 00000020:00200000:2.0:1787826407.793216:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 297 items remaining. 00000020:00200000:2.0:1787826407.793218:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 296 items remaining. 00000020:00200000:2.0:1787826407.794343:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 295 items remaining. 00000020:00200000:2.0:1787826407.794346:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 294 items remaining. 00000020:00200000:2.0:1787826407.794348:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 293 items remaining. 00000020:00200000:2.0:1787826407.794350:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 292 items remaining. 00000020:00200000:2.0:1787826407.794352:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 291 items remaining. 00000020:00200000:2.0:1787826407.794353:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 290 items remaining. 00000020:00200000:2.0:1787826407.794355:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 289 items remaining. 00000020:00200000:2.0:1787826407.794357:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 288 items remaining. 00000020:00200000:2.0:1787826407.794359:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 287 items remaining. 00000020:00200000:2.0:1787826407.794361:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 286 items remaining. 00000020:00200000:2.0:1787826407.794362:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 285 items remaining. 00000020:00200000:2.0:1787826407.794364:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 284 items remaining. 00000020:00200000:2.0:1787826407.794366:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 283 items remaining. 00000020:00200000:2.0:1787826407.794368:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 282 items remaining. 00000020:00200000:2.0:1787826407.794370:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 281 items remaining. 00000020:00200000:2.0:1787826407.794371:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 280 items remaining. 00000020:00200000:2.0:1787826407.794373:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 279 items remaining. 00000020:00200000:2.0:1787826407.794375:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 278 items remaining. 00000020:00200000:2.0:1787826407.794376:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 277 items remaining. 00000020:00200000:2.0:1787826407.794379:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 276 items remaining. 00000020:00200000:2.0:1787826407.794381:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 275 items remaining. 00000020:00200000:2.0:1787826407.794383:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 274 items remaining. 00000020:00200000:2.0:1787826407.794385:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 273 items remaining. 00000020:00200000:2.0:1787826407.794387:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 272 items remaining. 00000020:00200000:2.0:1787826407.794388:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 271 items remaining. 00000020:00200000:2.0:1787826407.794593:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 270 items remaining. 00000020:00200000:2.0:1787826407.794595:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 269 items remaining. 00000020:00200000:2.0:1787826407.794597:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 268 items remaining. 00000020:00200000:2.0:1787826407.794599:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 267 items remaining. 00000020:00200000:2.0:1787826407.794600:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 266 items remaining. 00000020:00200000:2.0:1787826407.794602:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 265 items remaining. 00000020:00200000:2.0:1787826407.794604:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 264 items remaining. 00000020:00200000:2.0:1787826407.794620:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 263 items remaining. 00000020:00200000:2.0:1787826407.794623:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 262 items remaining. 00000020:00200000:2.0:1787826407.794625:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 261 items remaining. 00000020:00200000:2.0:1787826407.794627:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 260 items remaining. 00000020:00200000:2.0:1787826407.794629:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 259 items remaining. 00000020:00200000:2.0:1787826407.794631:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 258 items remaining. 00000020:00200000:2.0:1787826407.794633:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 257 items remaining. 00000020:00200000:2.0:1787826407.794635:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000020:00200000:2.0:1787826407.794636:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1787826407.794638:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1787826407.794641:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1787826407.794642:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1787826407.794644:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1787826407.794646:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1787826407.794647:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1787826407.794649:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1787826407.794650:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1787826407.794652:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1787826407.794654:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1787826407.794656:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1787826407.794658:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1787826407.794659:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1787826407.794661:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1787826407.794663:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1787826407.794664:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1787826407.794666:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1787826407.794667:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1787826407.794669:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1787826407.794670:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1787826407.794672:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1787826407.794673:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1787826407.794675:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1787826407.794677:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1787826407.794679:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1787826407.794681:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1787826407.794682:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1787826407.794684:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1787826407.794685:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1787826407.794687:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1787826407.794689:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1787826407.794691:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1787826407.794692:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1787826407.794694:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1787826407.794696:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1787826407.794698:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1787826407.794699:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1787826407.794701:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1787826407.794703:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1787826407.794704:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1787826407.794706:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1787826407.794707:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1787826407.794709:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1787826407.794711:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1787826407.794713:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1787826407.794715:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1787826407.794717:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1787826407.794719:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1787826407.794720:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1787826407.794722:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1787826407.794723:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1787826407.794725:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1787826407.794727:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1787826407.794728:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1787826407.795014:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1787826407.795017:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1787826407.795019:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1787826407.795021:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1787826407.795023:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1787826407.795025:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1787826407.795027:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1787826407.795028:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1787826407.795030:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1787826407.795032:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1787826407.795034:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1787826407.795036:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1787826407.795037:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1787826407.795039:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1787826407.795041:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1787826407.795043:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1787826407.795044:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1787826407.795046:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1787826407.795047:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1787826407.795049:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1787826407.795051:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1787826407.795052:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1787826407.795054:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1787826407.795056:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1787826407.795057:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1787826407.795058:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1787826407.795060:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1787826407.795061:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1787826407.795063:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1787826407.795064:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1787826407.795066:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1787826407.795068:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1787826407.795069:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1787826407.795070:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1787826407.795072:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1787826407.795073:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1787826407.795074:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1787826407.795076:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1787826407.795078:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1787826407.795079:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1787826407.795081:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1787826407.795082:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1787826407.795083:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1787826407.795085:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1787826407.795086:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1787826407.795088:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1787826407.795090:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1787826407.795091:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1787826407.795094:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1787826407.795095:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1787826407.795096:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1787826407.795098:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1787826407.795100:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1787826407.795102:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1787826407.795103:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1787826407.795105:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1787826407.795106:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1787826407.795108:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1787826407.795110:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1787826407.795111:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1787826407.795113:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1787826407.795115:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1787826407.795117:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1787826407.795119:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1787826407.795120:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1787826407.795122:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1787826407.795124:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1787826407.795126:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1787826407.795127:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1787826407.795129:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1787826407.795131:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1787826407.795132:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1787826407.795135:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1787826407.795136:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1787826407.795138:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1787826407.795140:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1787826407.795142:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1787826407.795143:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1787826407.795145:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1787826407.795147:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1787826407.795149:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1787826407.795150:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1787826407.795152:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1787826407.795153:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1787826407.795155:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1787826407.795157:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1787826407.795159:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1787826407.795160:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1787826407.795161:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1787826407.795163:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1787826407.795164:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1787826407.795166:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1787826407.795168:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1787826407.795169:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1787826407.795171:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1787826407.795172:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1787826407.795174:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1787826407.795176:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1787826407.795178:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1787826407.795179:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1787826407.795181:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1787826407.795183:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1787826407.795184:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1787826407.795186:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1787826407.795188:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1787826407.795190:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1787826407.795192:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1787826407.795193:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1787826407.795195:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1787826407.795196:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1787826407.795198:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1787826407.795200:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1787826407.795201:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1787826407.795203:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1787826407.795205:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1787826407.795207:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1787826407.795209:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1787826407.795211:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1787826407.795213:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1787826407.795215:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1787826407.795217:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1787826407.795218:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1787826407.795923:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1787826407.795925:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1787826407.795927:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1787826407.795929:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1787826407.795931:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1787826407.795932:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1787826407.795934:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1787826407.795936:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1787826407.795938:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1787826407.795940:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1787826407.795941:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1787826407.795943:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1787826407.795945:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1787826407.795947:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1787826407.795948:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1787826407.795950:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1787826407.795952:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1787826407.795954:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1787826407.795956:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1787826407.795958:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1787826407.795959:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1787826407.795962:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1787826407.795963:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1787826407.795965:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1787826407.795967:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1787826407.795970:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1787826407.795973:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1787826407.795975:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1787826407.795976:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1787826407.795978:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1787826407.795979:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1787826407.795981:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1787826407.795983:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1787826407.795984:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1787826407.795986:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1787826407.795988:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1787826407.795989:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1787826407.795991:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1787826407.795993:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1787826407.795994:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1787826407.795996:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1787826407.795997:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1787826407.795999:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1787826407.796001:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1787826407.796003:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1787826407.796005:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1787826407.796006:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1787826407.796008:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1787826407.796009:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1787826407.796011:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1787826407.796013:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1787826407.796014:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1787826407.796016:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1787826407.796018:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1787826407.796020:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1787826407.796022:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1787826407.796024:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1787826407.796025:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1787826407.796028:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1787826407.796030:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1787826407.796032:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1787826407.796033:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1787826407.796035:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1787826407.796037:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1787826407.796038:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1787826407.796040:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1787826407.796042:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1787826407.796043:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1787826407.796045:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1787826407.796047:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1787826407.796049:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1787826407.796050:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1787826407.796052:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1787826407.796054:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1787826407.796055:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1787826407.796056:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1787826407.796058:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1787826407.796060:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1787826407.796061:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1787826407.798146:0:2350:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1787826407.798261:0:2350:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907d83b79540 00000100:00100000:2.0:1787826407.798295:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011472512:192.168.203.104@tcp:3:lfs.0 00000080:00200000:0.0:1787826407.801995:0:69451:0:(file.c:2207:ll_file_io_generic()) Range unlock [0, 511] 00000080:00200000:0.0:1787826407.802015:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1787826407.802022:0:69451:0:(file.c:2240:ll_file_io_generic()) f204e.sanity-flr: 1 io complete with rc: 0, result: 2097152, restart: 0 00000020:00200000:0.0:1787826407.802027:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1787826407.802042:0:69451:0:(file.c:2326:ll_file_io_generic()) iot: 1, result: 2097152 00000080:00200080:0.0:1787826407.802062:0:69451:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8) cmd=400866b1 arg=0 00000080:00200080:0.0:1787826407.802074:0:69451:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8) cmd=400866b1 arg=2 00000080:00200008:0.0:1787826407.802081:0:69451:0:(file.c:6313:ll_getattr_dentry()) START file f204e.sanity-flr:[0x200000404:0x209:0x0](ffff907d933121c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1787826407.802090:0:69451:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8),name=f204e.sanity-flr 00000002:00010000:0.0:1787826407.802112:0:69451:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x209:0x0]) in obj [0x200000404:0x209:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826407.802132:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826407.802148:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:0.0:1787826407.802158:0:69451:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x209:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826407.802166:0:69451:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) 00000080:00010000:0.0:1787826407.802170:0:69451:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) for lock 0xe5f736940357bb21 00000080:00010000:0.0:1787826407.802179:0:69451:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357bb21 from it ffff9dd142bd7cd8 00010000:00010000:0.0:1787826407.802184:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1787826407.802194:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1787826407.802218:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:0.0:1787826407.803078:0:69451:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1787826407.803083:0:69451:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:0.0:1787826407.803087:0:69451:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x209:0x0] 00000008:00010000:0.0:1787826407.803110:0:69451:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1787826407.803114:0:69451:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da032e9b0, osc lock ffff907d96368000, flags 0x400000400001002 00010000:00010000:0.0:1787826407.803135:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1787826407.803151:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 104857599) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:0.0:1787826407.803165:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1787826407.803169:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:0.0:1787826407.803178:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00200000:0.0:1787826407.803191:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1787826407.803201:0:69451:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x209:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:0.0:1787826407.803206:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5792c70 (ffff907d884a7d88/0) at cl_lock_release():245 00000020:00010000:0.0:1787826407.803209:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5792c70 (ffff907d884a7d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1787826407.803213:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da032e9b0 (ffff907d884a71b8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1787826407.803217:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5792c70 (ffff907d884a7d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1787826407.803814:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da032e9b0 (ffff907d884a7d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1787826407.803974:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1787826407.804007:0:69451:0:(file.c:2652:do_file_write_iter()) START file f204e.sanity-flr:[0x200000404:0x209:0x0], ppos: 0, count: 2097152 00000080:00200000:0.0:1787826407.804014:0:69451:0:(file.c:2090:ll_file_io_generic()) f204e.sanity-flr: write ppos: 0, bytes: 2097152 00000080:00200000:0.0:1787826407.804024:0:69451:0:(file.c:1848:ll_io_set_mirror()) f204e.sanity-flr: desiginated mirror: 2 00000020:00200000:0.0:1787826407.804034:0:69451:0:(cl_io.c:218:cl_io_rw_init()) header@0000000033ed2c48[0x110, 2, [0x200000404:0x209:0x0] hash] 00000020:00200000:0.0:1787826407.804036:0:69451:0:(cl_io.c:218:cl_io_rw_init()) io range: 2 [0, 2097152) 2048 0 00000080:00200000:0.0:1787826407.804040:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00000080:00200000:0.0:1787826407.804051:0:69451:0:(file.c:2170:ll_file_io_generic()) Range lock [0, 511] 00020000:00200000:0.0:1787826407.804058:0:69451:0:(lov_io.c:1445:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 2097152) 2097152, 2097152 00020000:00200000:0.0:1787826407.804061:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x11 00000080:00200000:0.0:1787826407.804074:0:69451:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 511] 00000008:00010000:0.0:1787826407.804084:0:69451:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 11 00000008:00010000:0.0:1787826407.804089:0:69451:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da032e9b0, osc lock ffff907d96368000, flags 0x40000 00000080:00200000:0.0:1787826407.804096:0:69451:0:(vvp_io.c:1339:vvp_io_write_start()) f204e.sanity-flr: write [0, 2097152) append 0 append lockpos 0 00000080:00200000:0.0:1787826407.804124:0:69451:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8), size=2097152 (max 335544320), offset=0=0x0, pages 512 (max 81920), lockless, parallel 00000020:00200000:0.0:1787826407.806149:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 512 items remaining. 00000080:00200000:0.0:1787826407.806158:0:69451:0:(rw26.c:550:ll_direct_IO()) result 0 tot_bytes 2097152 count 2097152 file_offset 2097152 00000080:00200000:0.0:1787826407.806174:0:69451:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x209:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:0.0:1787826407.806179:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_release():245 00000020:00010000:0.0:1787826407.806182:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1787826407.806185:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da032e9b0 (ffff907d884a71b8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1787826407.806188:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1787826407.806190:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da032e9b0 (ffff907d884a7d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1787826407.806198:0:69451:0:(vvp_io.c:533:vvp_io_advance()) advancing 2097152 bytes 00000020:00200000:0.0:1787826407.806205:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1787826407.806684:0:26204:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff907d9114f600 00000008:00100000:0.0:1787826407.809326:0:26204:0:(osc_request.c:2124:osc_brw_prep_request()) brw rpc ffff907d90845a40 - object 0x240000400:810 offset 0<>2097152 00000100:00100000:1.0:1787826407.809434:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011472640:192.168.203.104@tcp:4:lfs.0 00000100:00100000:1.0:1787826407.880783:0:2349:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff907d986d0000: committing for last_committed 21474840523 gen 3 00000008:00010000:1.0:1787826407.880858:0:2349:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff907d9114f600 00000020:00200000:1.0:1787826407.880866:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 511 items remaining. 00000020:00200000:1.0:1787826407.880868:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 510 items remaining. 00000020:00200000:1.0:1787826407.880870:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 509 items remaining. 00000020:00200000:1.0:1787826407.880872:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 508 items remaining. 00000020:00200000:1.0:1787826407.880874:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 507 items remaining. 00000020:00200000:1.0:1787826407.880875:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 506 items remaining. 00000020:00200000:1.0:1787826407.880877:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 505 items remaining. 00000020:00200000:1.0:1787826407.880878:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 504 items remaining. 00000020:00200000:1.0:1787826407.880880:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 503 items remaining. 00000020:00200000:1.0:1787826407.880882:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 502 items remaining. 00000020:00200000:1.0:1787826407.880883:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 501 items remaining. 00000020:00200000:1.0:1787826407.880885:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 500 items remaining. 00000020:00200000:1.0:1787826407.880887:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 499 items remaining. 00000020:00200000:1.0:1787826407.880888:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 498 items remaining. 00000020:00200000:1.0:1787826407.880889:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 497 items remaining. 00000020:00200000:1.0:1787826407.880891:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 496 items remaining. 00000020:00200000:1.0:1787826407.880892:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 495 items remaining. 00000020:00200000:1.0:1787826407.880894:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 494 items remaining. 00000020:00200000:1.0:1787826407.880896:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 493 items remaining. 00000020:00200000:1.0:1787826407.880897:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 492 items remaining. 00000020:00200000:1.0:1787826407.880899:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 491 items remaining. 00000020:00200000:1.0:1787826407.880900:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 490 items remaining. 00000020:00200000:1.0:1787826407.880902:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 489 items remaining. 00000020:00200000:1.0:1787826407.880903:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 488 items remaining. 00000020:00200000:1.0:1787826407.880905:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 487 items remaining. 00000020:00200000:1.0:1787826407.880906:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 486 items remaining. 00000020:00200000:1.0:1787826407.880907:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 485 items remaining. 00000020:00200000:1.0:1787826407.880909:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 484 items remaining. 00000020:00200000:1.0:1787826407.880910:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 483 items remaining. 00000020:00200000:1.0:1787826407.880912:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 482 items remaining. 00000020:00200000:1.0:1787826407.880914:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 481 items remaining. 00000020:00200000:1.0:1787826407.880915:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 480 items remaining. 00000020:00200000:1.0:1787826407.880916:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 479 items remaining. 00000020:00200000:1.0:1787826407.880918:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 478 items remaining. 00000020:00200000:1.0:1787826407.880919:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 477 items remaining. 00000020:00200000:1.0:1787826407.880921:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 476 items remaining. 00000020:00200000:1.0:1787826407.880923:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 475 items remaining. 00000020:00200000:1.0:1787826407.880925:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 474 items remaining. 00000020:00200000:1.0:1787826407.880927:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 473 items remaining. 00000020:00200000:1.0:1787826407.880929:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 472 items remaining. 00000020:00200000:1.0:1787826407.880930:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 471 items remaining. 00000020:00200000:1.0:1787826407.880943:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 470 items remaining. 00000020:00200000:1.0:1787826407.880946:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 469 items remaining. 00000020:00200000:1.0:1787826407.880947:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 468 items remaining. 00000020:00200000:1.0:1787826407.880949:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 467 items remaining. 00000020:00200000:1.0:1787826407.880950:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 466 items remaining. 00000020:00200000:1.0:1787826407.880952:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 465 items remaining. 00000020:00200000:1.0:1787826407.880953:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 464 items remaining. 00000020:00200000:1.0:1787826407.880955:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 463 items remaining. 00000020:00200000:1.0:1787826407.880956:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 462 items remaining. 00000020:00200000:1.0:1787826407.880958:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 461 items remaining. 00000020:00200000:1.0:1787826407.880959:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 460 items remaining. 00000020:00200000:1.0:1787826407.880961:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 459 items remaining. 00000020:00200000:1.0:1787826407.880962:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 458 items remaining. 00000020:00200000:1.0:1787826407.880964:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 457 items remaining. 00000020:00200000:1.0:1787826407.880965:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 456 items remaining. 00000020:00200000:1.0:1787826407.880967:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 455 items remaining. 00000020:00200000:1.0:1787826407.880969:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 454 items remaining. 00000020:00200000:1.0:1787826407.880970:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 453 items remaining. 00000020:00200000:1.0:1787826407.880972:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 452 items remaining. 00000020:00200000:1.0:1787826407.880973:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 451 items remaining. 00000020:00200000:1.0:1787826407.880975:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 450 items remaining. 00000020:00200000:1.0:1787826407.880976:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 449 items remaining. 00000020:00200000:1.0:1787826407.880978:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 448 items remaining. 00000020:00200000:1.0:1787826407.880979:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 447 items remaining. 00000020:00200000:1.0:1787826407.880981:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 446 items remaining. 00000020:00200000:1.0:1787826407.880982:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 445 items remaining. 00000020:00200000:1.0:1787826407.880984:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 444 items remaining. 00000020:00200000:1.0:1787826407.880987:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 443 items remaining. 00000020:00200000:1.0:1787826407.880988:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 442 items remaining. 00000020:00200000:1.0:1787826407.880989:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 441 items remaining. 00000020:00200000:1.0:1787826407.880991:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 440 items remaining. 00000020:00200000:1.0:1787826407.880993:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 439 items remaining. 00000020:00200000:1.0:1787826407.880994:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 438 items remaining. 00000020:00200000:1.0:1787826407.880995:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 437 items remaining. 00000020:00200000:1.0:1787826407.880997:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 436 items remaining. 00000020:00200000:1.0:1787826407.880998:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 435 items remaining. 00000020:00200000:1.0:1787826407.880999:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 434 items remaining. 00000020:00200000:1.0:1787826407.881001:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 433 items remaining. 00000020:00200000:1.0:1787826407.881003:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 432 items remaining. 00000020:00200000:1.0:1787826407.881005:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 431 items remaining. 00000020:00200000:1.0:1787826407.881006:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 430 items remaining. 00000020:00200000:1.0:1787826407.881007:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 429 items remaining. 00000020:00200000:1.0:1787826407.881009:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 428 items remaining. 00000020:00200000:1.0:1787826407.881010:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 427 items remaining. 00000020:00200000:1.0:1787826407.881011:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 426 items remaining. 00000020:00200000:1.0:1787826407.881013:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 425 items remaining. 00000020:00200000:1.0:1787826407.881014:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 424 items remaining. 00000020:00200000:1.0:1787826407.881016:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 423 items remaining. 00000020:00200000:1.0:1787826407.881017:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 422 items remaining. 00000020:00200000:1.0:1787826407.881019:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 421 items remaining. 00000020:00200000:1.0:1787826407.881021:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 420 items remaining. 00000020:00200000:1.0:1787826407.881023:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 419 items remaining. 00000020:00200000:1.0:1787826407.881024:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 418 items remaining. 00000020:00200000:1.0:1787826407.881025:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 417 items remaining. 00000020:00200000:1.0:1787826407.881027:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 416 items remaining. 00000020:00200000:1.0:1787826407.881028:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 415 items remaining. 00000020:00200000:1.0:1787826407.881030:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 414 items remaining. 00000020:00200000:1.0:1787826407.881031:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 413 items remaining. 00000020:00200000:1.0:1787826407.881033:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 412 items remaining. 00000020:00200000:1.0:1787826407.881035:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 411 items remaining. 00000020:00200000:1.0:1787826407.881037:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 410 items remaining. 00000020:00200000:1.0:1787826407.881038:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 409 items remaining. 00000020:00200000:1.0:1787826407.881040:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 408 items remaining. 00000020:00200000:1.0:1787826407.881041:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 407 items remaining. 00000020:00200000:1.0:1787826407.881043:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 406 items remaining. 00000020:00200000:1.0:1787826407.881045:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 405 items remaining. 00000020:00200000:1.0:1787826407.881046:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 404 items remaining. 00000020:00200000:1.0:1787826407.881048:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 403 items remaining. 00000020:00200000:1.0:1787826407.881049:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 402 items remaining. 00000020:00200000:1.0:1787826407.881050:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 401 items remaining. 00000020:00200000:1.0:1787826407.881052:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 400 items remaining. 00000020:00200000:1.0:1787826407.881053:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 399 items remaining. 00000020:00200000:1.0:1787826407.881055:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 398 items remaining. 00000020:00200000:1.0:1787826407.881056:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 397 items remaining. 00000020:00200000:1.0:1787826407.881058:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 396 items remaining. 00000020:00200000:1.0:1787826407.881060:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 395 items remaining. 00000020:00200000:1.0:1787826407.881062:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 394 items remaining. 00000020:00200000:1.0:1787826407.881064:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 393 items remaining. 00000020:00200000:1.0:1787826407.881065:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 392 items remaining. 00000020:00200000:1.0:1787826407.881067:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 391 items remaining. 00000020:00200000:1.0:1787826407.881068:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 390 items remaining. 00000020:00200000:1.0:1787826407.881070:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 389 items remaining. 00000020:00200000:1.0:1787826407.881071:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 388 items remaining. 00000020:00200000:1.0:1787826407.881073:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 387 items remaining. 00000020:00200000:1.0:1787826407.881075:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 386 items remaining. 00000020:00200000:1.0:1787826407.881076:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 385 items remaining. 00000020:00200000:1.0:1787826407.881077:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 384 items remaining. 00000020:00200000:1.0:1787826407.881079:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 383 items remaining. 00000020:00200000:1.0:1787826407.881080:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 382 items remaining. 00000020:00200000:1.0:1787826407.881082:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 381 items remaining. 00000020:00200000:1.0:1787826407.881083:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 380 items remaining. 00000020:00200000:1.0:1787826407.881085:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 379 items remaining. 00000020:00200000:1.0:1787826407.881087:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 378 items remaining. 00000020:00200000:1.0:1787826407.881088:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 377 items remaining. 00000020:00200000:1.0:1787826407.881090:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 376 items remaining. 00000020:00200000:1.0:1787826407.881091:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 375 items remaining. 00000020:00200000:1.0:1787826407.881093:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 374 items remaining. 00000020:00200000:1.0:1787826407.881095:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 373 items remaining. 00000020:00200000:1.0:1787826407.881096:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 372 items remaining. 00000020:00200000:1.0:1787826407.881097:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 371 items remaining. 00000020:00200000:1.0:1787826407.881099:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 370 items remaining. 00000020:00200000:1.0:1787826407.881100:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 369 items remaining. 00000020:00200000:1.0:1787826407.881102:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 368 items remaining. 00000020:00200000:1.0:1787826407.881104:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 367 items remaining. 00000020:00200000:1.0:1787826407.881105:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 366 items remaining. 00000020:00200000:1.0:1787826407.881107:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 365 items remaining. 00000020:00200000:1.0:1787826407.881108:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 364 items remaining. 00000020:00200000:1.0:1787826407.881110:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 363 items remaining. 00000020:00200000:1.0:1787826407.881112:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 362 items remaining. 00000020:00200000:1.0:1787826407.881113:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 361 items remaining. 00000020:00200000:1.0:1787826407.881115:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 360 items remaining. 00000020:00200000:1.0:1787826407.881117:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 359 items remaining. 00000020:00200000:1.0:1787826407.881118:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 358 items remaining. 00000020:00200000:1.0:1787826407.881120:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 357 items remaining. 00000020:00200000:1.0:1787826407.881122:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 356 items remaining. 00000020:00200000:1.0:1787826407.881123:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 355 items remaining. 00000020:00200000:1.0:1787826407.881125:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 354 items remaining. 00000020:00200000:1.0:1787826407.881126:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 353 items remaining. 00000020:00200000:1.0:1787826407.881128:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 352 items remaining. 00000020:00200000:1.0:1787826407.881129:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 351 items remaining. 00000020:00200000:1.0:1787826407.881131:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 350 items remaining. 00000020:00200000:1.0:1787826407.881133:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 349 items remaining. 00000020:00200000:1.0:1787826407.881134:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 348 items remaining. 00000020:00200000:1.0:1787826407.881136:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 347 items remaining. 00000020:00200000:1.0:1787826407.881138:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 346 items remaining. 00000020:00200000:1.0:1787826407.881140:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 345 items remaining. 00000020:00200000:1.0:1787826407.881141:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 344 items remaining. 00000020:00200000:1.0:1787826407.881143:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 343 items remaining. 00000020:00200000:1.0:1787826407.881144:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 342 items remaining. 00000020:00200000:1.0:1787826407.881146:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 341 items remaining. 00000020:00200000:1.0:1787826407.881147:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 340 items remaining. 00000020:00200000:1.0:1787826407.881149:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 339 items remaining. 00000020:00200000:1.0:1787826407.881151:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 338 items remaining. 00000020:00200000:1.0:1787826407.881152:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 337 items remaining. 00000020:00200000:1.0:1787826407.881154:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 336 items remaining. 00000020:00200000:1.0:1787826407.881156:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 335 items remaining. 00000020:00200000:1.0:1787826407.881157:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 334 items remaining. 00000020:00200000:1.0:1787826407.881159:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 333 items remaining. 00000020:00200000:1.0:1787826407.881161:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 332 items remaining. 00000020:00200000:1.0:1787826407.881162:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 331 items remaining. 00000020:00200000:1.0:1787826407.881164:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 330 items remaining. 00000020:00200000:1.0:1787826407.881165:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 329 items remaining. 00000020:00200000:1.0:1787826407.881167:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 328 items remaining. 00000020:00200000:1.0:1787826407.881168:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 327 items remaining. 00000020:00200000:1.0:1787826407.881170:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 326 items remaining. 00000020:00200000:1.0:1787826407.881172:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 325 items remaining. 00000020:00200000:1.0:1787826407.881174:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 324 items remaining. 00000020:00200000:1.0:1787826407.881175:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 323 items remaining. 00000020:00200000:1.0:1787826407.881177:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 322 items remaining. 00000020:00200000:1.0:1787826407.881178:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 321 items remaining. 00000020:00200000:1.0:1787826407.881179:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 320 items remaining. 00000020:00200000:1.0:1787826407.881181:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 319 items remaining. 00000020:00200000:1.0:1787826407.881182:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 318 items remaining. 00000020:00200000:1.0:1787826407.881184:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 317 items remaining. 00000020:00200000:1.0:1787826407.881185:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 316 items remaining. 00000020:00200000:1.0:1787826407.881187:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 315 items remaining. 00000020:00200000:1.0:1787826407.881189:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 314 items remaining. 00000020:00200000:1.0:1787826407.881191:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 313 items remaining. 00000020:00200000:1.0:1787826407.881192:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 312 items remaining. 00000020:00200000:1.0:1787826407.881194:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 311 items remaining. 00000020:00200000:1.0:1787826407.881196:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 310 items remaining. 00000020:00200000:1.0:1787826407.881198:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 309 items remaining. 00000020:00200000:1.0:1787826407.881199:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 308 items remaining. 00000020:00200000:1.0:1787826407.881201:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 307 items remaining. 00000020:00200000:1.0:1787826407.881202:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 306 items remaining. 00000020:00200000:1.0:1787826407.881204:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 305 items remaining. 00000020:00200000:1.0:1787826407.881206:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 304 items remaining. 00000020:00200000:1.0:1787826407.881207:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 303 items remaining. 00000020:00200000:1.0:1787826407.881209:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 302 items remaining. 00000020:00200000:1.0:1787826407.881211:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 301 items remaining. 00000020:00200000:1.0:1787826407.881213:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 300 items remaining. 00000020:00200000:1.0:1787826407.881214:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 299 items remaining. 00000020:00200000:1.0:1787826407.881216:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 298 items remaining. 00000020:00200000:1.0:1787826407.881217:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 297 items remaining. 00000020:00200000:1.0:1787826407.881219:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 296 items remaining. 00000020:00200000:1.0:1787826407.881220:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 295 items remaining. 00000020:00200000:1.0:1787826407.881222:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 294 items remaining. 00000020:00200000:1.0:1787826407.881223:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 293 items remaining. 00000020:00200000:1.0:1787826407.881225:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 292 items remaining. 00000020:00200000:1.0:1787826407.881227:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 291 items remaining. 00000020:00200000:1.0:1787826407.881228:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 290 items remaining. 00000020:00200000:1.0:1787826407.881229:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 289 items remaining. 00000020:00200000:1.0:1787826407.881231:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 288 items remaining. 00000020:00200000:1.0:1787826407.881233:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 287 items remaining. 00000020:00200000:1.0:1787826407.881235:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 286 items remaining. 00000020:00200000:1.0:1787826407.881236:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 285 items remaining. 00000020:00200000:1.0:1787826407.881237:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 284 items remaining. 00000020:00200000:1.0:1787826407.881240:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 283 items remaining. 00000020:00200000:1.0:1787826407.881241:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 282 items remaining. 00000020:00200000:1.0:1787826407.881243:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 281 items remaining. 00000020:00200000:1.0:1787826407.881245:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 280 items remaining. 00000020:00200000:1.0:1787826407.881246:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 279 items remaining. 00000020:00200000:1.0:1787826407.881248:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 278 items remaining. 00000020:00200000:1.0:1787826407.881250:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 277 items remaining. 00000020:00200000:1.0:1787826407.881251:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 276 items remaining. 00000020:00200000:1.0:1787826407.881253:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 275 items remaining. 00000020:00200000:1.0:1787826407.881255:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 274 items remaining. 00000020:00200000:1.0:1787826407.881256:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 273 items remaining. 00000020:00200000:1.0:1787826407.881258:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 272 items remaining. 00000020:00200000:1.0:1787826407.881260:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 271 items remaining. 00000020:00200000:1.0:1787826407.881261:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 270 items remaining. 00000020:00200000:1.0:1787826407.881263:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 269 items remaining. 00000020:00200000:1.0:1787826407.881264:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 268 items remaining. 00000020:00200000:1.0:1787826407.881266:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 267 items remaining. 00000020:00200000:1.0:1787826407.881267:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 266 items remaining. 00000020:00200000:1.0:1787826407.881269:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 265 items remaining. 00000020:00200000:1.0:1787826407.881270:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 264 items remaining. 00000020:00200000:1.0:1787826407.881272:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 263 items remaining. 00000020:00200000:1.0:1787826407.881273:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 262 items remaining. 00000020:00200000:1.0:1787826407.881275:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 261 items remaining. 00000020:00200000:1.0:1787826407.881277:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 260 items remaining. 00000020:00200000:1.0:1787826407.881278:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 259 items remaining. 00000020:00200000:1.0:1787826407.881279:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 258 items remaining. 00000020:00200000:1.0:1787826407.881281:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 257 items remaining. 00000020:00200000:1.0:1787826407.881283:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000020:00200000:1.0:1787826407.881284:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:1.0:1787826407.881286:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:1.0:1787826407.881287:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:1.0:1787826407.881289:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:1.0:1787826407.881291:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:1.0:1787826407.881292:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:1.0:1787826407.881294:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:1.0:1787826407.881296:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:1.0:1787826407.881297:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:1.0:1787826407.881299:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:1.0:1787826407.881300:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:1.0:1787826407.881302:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:1.0:1787826407.881304:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:1.0:1787826407.881305:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:1.0:1787826407.881307:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:1.0:1787826407.881309:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:1.0:1787826407.881310:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:1.0:1787826407.881312:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:1.0:1787826407.881314:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:1.0:1787826407.881315:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:1.0:1787826407.881317:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:1.0:1787826407.881318:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:1.0:1787826407.881319:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:1.0:1787826407.881321:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:1.0:1787826407.881334:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:1.0:1787826407.881335:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:1.0:1787826407.881337:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:1.0:1787826407.881339:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:1.0:1787826407.881341:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:1.0:1787826407.881342:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:1.0:1787826407.881344:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:1.0:1787826407.881345:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:1.0:1787826407.881347:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:1.0:1787826407.881348:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:1.0:1787826407.881350:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:1.0:1787826407.881351:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:1.0:1787826407.881353:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:1.0:1787826407.881355:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:1.0:1787826407.881357:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:1.0:1787826407.881358:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:1.0:1787826407.881360:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:1.0:1787826407.881361:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:1.0:1787826407.881363:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:1.0:1787826407.881365:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:1.0:1787826407.881366:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:1.0:1787826407.881367:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:1.0:1787826407.881369:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:1.0:1787826407.881371:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:1.0:1787826407.881372:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:1.0:1787826407.881374:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:1.0:1787826407.881375:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:1.0:1787826407.881377:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:1.0:1787826407.881378:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:1.0:1787826407.881379:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:1.0:1787826407.881381:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:1.0:1787826407.881383:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:1.0:1787826407.881384:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:1.0:1787826407.881386:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:1.0:1787826407.881388:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:1.0:1787826407.881389:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:1.0:1787826407.881391:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:1.0:1787826407.881392:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:1.0:1787826407.881394:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:1.0:1787826407.881395:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:1.0:1787826407.881397:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:1.0:1787826407.881398:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:1.0:1787826407.881400:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:1.0:1787826407.881401:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:1.0:1787826407.881403:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:1.0:1787826407.881405:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:1.0:1787826407.881406:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:1.0:1787826407.881408:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:1.0:1787826407.881409:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:1.0:1787826407.881410:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:1.0:1787826407.881412:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:1.0:1787826407.881414:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:1.0:1787826407.881415:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:1.0:1787826407.881416:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:1.0:1787826407.881418:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:1.0:1787826407.881419:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:1.0:1787826407.881421:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:1.0:1787826407.881423:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:1.0:1787826407.881425:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:1.0:1787826407.881427:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:1.0:1787826407.881429:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:1.0:1787826407.881431:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:1.0:1787826407.881433:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:1.0:1787826407.881434:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:1.0:1787826407.881436:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:1.0:1787826407.881438:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:1.0:1787826407.881439:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:1.0:1787826407.881440:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:1.0:1787826407.881442:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:1.0:1787826407.881443:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:1.0:1787826407.881445:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:1.0:1787826407.881446:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:1.0:1787826407.881448:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:1.0:1787826407.881449:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:1.0:1787826407.881450:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:1.0:1787826407.881584:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:1.0:1787826407.881588:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:1.0:1787826407.881590:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:1.0:1787826407.881593:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:1.0:1787826407.881594:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:1.0:1787826407.881596:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:1.0:1787826407.881598:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:1.0:1787826407.881599:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:1.0:1787826407.881601:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:1.0:1787826407.881602:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:1.0:1787826407.881604:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:1.0:1787826407.881642:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:1.0:1787826407.881644:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:1.0:1787826407.881646:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:1.0:1787826407.881648:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:1.0:1787826407.881650:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:1.0:1787826407.881652:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:1.0:1787826407.881654:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:1.0:1787826407.881656:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:1.0:1787826407.881658:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:1.0:1787826407.881659:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:1.0:1787826407.881661:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:1.0:1787826407.881663:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:1.0:1787826407.881665:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:1.0:1787826407.881666:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:1.0:1787826407.881668:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:1.0:1787826407.881669:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:1.0:1787826407.881671:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:1.0:1787826407.881673:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:1.0:1787826407.881674:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:1.0:1787826407.881676:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:1.0:1787826407.881677:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:1.0:1787826407.881679:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:1.0:1787826407.881681:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:1.0:1787826407.881683:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:1.0:1787826407.881684:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:1.0:1787826407.881686:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:1.0:1787826407.881688:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:1.0:1787826407.881689:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:1.0:1787826407.881690:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:1.0:1787826407.881692:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:1.0:1787826407.881693:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:1.0:1787826407.881695:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:1.0:1787826407.881696:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:1.0:1787826407.881697:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:1.0:1787826407.881698:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:1.0:1787826407.881700:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:1.0:1787826407.881702:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:1.0:1787826407.881703:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:1.0:1787826407.881705:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:1.0:1787826407.881707:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:1.0:1787826407.881708:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:1.0:1787826407.881710:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:1.0:1787826407.881712:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:1.0:1787826407.881714:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:1.0:1787826407.881716:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:1.0:1787826407.881718:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:1.0:1787826407.881719:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:1.0:1787826407.881721:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:1.0:1787826407.881722:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:1.0:1787826407.881724:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:1.0:1787826407.881725:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:1.0:1787826407.881727:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:1.0:1787826407.881729:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:1.0:1787826407.881731:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:1.0:1787826407.881733:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:1.0:1787826407.881735:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:1.0:1787826407.881736:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:1.0:1787826407.881738:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:1.0:1787826407.881740:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:1.0:1787826407.881742:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:1.0:1787826407.881744:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:1.0:1787826407.881745:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:1.0:1787826407.881747:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:1.0:1787826407.881749:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:1.0:1787826407.881750:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:1.0:1787826407.881752:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:1.0:1787826407.881753:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:1.0:1787826407.881755:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:1.0:1787826407.881756:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:1.0:1787826407.881758:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:1.0:1787826407.881760:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:1.0:1787826407.881762:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:1.0:1787826407.881763:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:1.0:1787826407.881765:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:1.0:1787826407.881767:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:1.0:1787826407.881768:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:1.0:1787826407.881770:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:1.0:1787826407.881772:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:1.0:1787826407.881774:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:1.0:1787826407.881776:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:1.0:1787826407.881778:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:1.0:1787826407.881780:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:1.0:1787826407.881782:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:1.0:1787826407.881783:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:1.0:1787826407.881785:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:1.0:1787826407.881787:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:1.0:1787826407.881788:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:1.0:1787826407.881791:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:1.0:1787826407.881793:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:1.0:1787826407.881794:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:1.0:1787826407.881796:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:1.0:1787826407.881798:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:1.0:1787826407.881799:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:1.0:1787826407.881801:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:1.0:1787826407.881803:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:1.0:1787826407.881805:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:1.0:1787826407.881806:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:1.0:1787826407.881809:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:1.0:1787826407.881811:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:1.0:1787826407.881812:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:1.0:1787826407.881814:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:1.0:1787826407.881816:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:1.0:1787826407.881818:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:1.0:1787826407.881820:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:1.0:1787826407.881822:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:1.0:1787826407.881824:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:1.0:1787826407.881825:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:1.0:1787826407.881827:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:1.0:1787826407.881828:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:1.0:1787826407.881830:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:1.0:1787826407.881831:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:1.0:1787826407.881833:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:1.0:1787826407.881834:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:1.0:1787826407.881836:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:1.0:1787826407.881837:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:1.0:1787826407.881839:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:1.0:1787826407.881841:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:1.0:1787826407.881842:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:1.0:1787826407.881844:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:1.0:1787826407.881846:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:1.0:1787826407.881848:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:1.0:1787826407.881850:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:1.0:1787826407.881852:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:1.0:1787826407.881853:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:1.0:1787826407.881855:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:1.0:1787826407.881856:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:1.0:1787826407.881858:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:1.0:1787826407.881860:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:1.0:1787826407.881862:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:1.0:1787826407.881863:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:1.0:1787826407.881865:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:1.0:1787826407.881867:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:1.0:1787826407.881869:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:1.0:1787826407.881870:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:1.0:1787826407.881872:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:1.0:1787826407.881874:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:1.0:1787826407.881875:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:1.0:1787826407.881877:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:1.0:1787826407.881879:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:1.0:1787826407.881880:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:1.0:1787826407.881882:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:1.0:1787826407.881884:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:1.0:1787826407.881886:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:1.0:1787826407.881888:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:1.0:1787826407.881889:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:1.0:1787826407.881891:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1787826407.882074:0:2349:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1787826407.883620:0:2349:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff907d9114f600 00000100:00100000:1.0:1787826407.883663:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011472640:192.168.203.104@tcp:4:lfs.0 00000080:00200000:2.0:1787826407.896056:0:69451:0:(file.c:2207:ll_file_io_generic()) Range unlock [0, 511] 00000080:00200000:2.0:1787826407.896078:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826407.896084:0:69451:0:(file.c:2240:ll_file_io_generic()) f204e.sanity-flr: 2 io complete with rc: 0, result: 2097152, restart: 0 00000080:00200000:2.0:1787826407.896099:0:69451:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x209:0x0] updating i_size 2097152 i_blocks 4096 00000020:00200000:2.0:1787826407.896104:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826407.896119:0:69451:0:(file.c:2326:ll_file_io_generic()) iot: 2, result: 2097152 00000080:00200080:2.0:1787826407.896141:0:69451:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8) cmd=400866b1 arg=0 00000080:00200080:2.0:1787826407.896153:0:69451:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8) cmd=400866b1 arg=1 00000080:00200008:2.0:1787826407.896160:0:69451:0:(file.c:6313:ll_getattr_dentry()) START file f204e.sanity-flr:[0x200000404:0x209:0x0](ffff907d933121c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1787826407.896168:0:69451:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8),name=f204e.sanity-flr 00000002:00010000:2.0:1787826407.896191:0:69451:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x209:0x0]) in obj [0x200000404:0x209:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826407.896212:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826407.896227:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1787826407.896236:0:69451:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x209:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826407.896244:0:69451:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) 00000080:00010000:2.0:1787826407.896247:0:69451:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) for lock 0xe5f736940357bb21 00000080:00010000:2.0:1787826407.896254:0:69451:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357bb21 from it ffff9dd142bd7cd8 00010000:00010000:2.0:1787826407.896258:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1787826407.896267:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1787826407.896291:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:2.0:1787826407.896301:0:69451:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826407.896304:0:69451:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1787826407.896307:0:69451:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x209:0x0] 00000008:00010000:2.0:1787826407.896326:0:69451:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826407.896329:0:69451:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322030, osc lock ffff907d830a3380, flags 0x400000400001002 00010000:00010000:2.0:1787826407.896349:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1787826407.896367:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 104857599) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00200000:2.0:1787826407.896378:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826407.896382:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1787826407.896389:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000020:00200000:2.0:1787826407.896399:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826407.896404:0:69451:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x209:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826407.896408:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5792c70 (ffff907d884a7d88/0) at cl_lock_release():245 00000020:00010000:2.0:1787826407.896411:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5792c70 (ffff907d884a7d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826407.896414:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322030 (ffff907d884a71b8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826407.896418:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5792c70 (ffff907d884a7d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826407.896421:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322030 (ffff907d884a7d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826407.896429:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826407.896442:0:69451:0:(file.c:5224:ll_file_seek()) START file f204e.sanity-flr:[0x200000404:0x209:0x0], offset: 2097152, type: SEEK_DATA 00000080:00200000:2.0:1787826407.896448:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:2.0:1787826407.896452:0:69451:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826407.896455:0:69451:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1787826407.896459:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1787826407.896463:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:2.0:1787826407.896485:0:69451:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x209:0x0] updating i_size 2097152 i_blocks 4096 00000080:00200000:2.0:1787826407.896494:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1787826407.896498:0:69451:0:(file.c:1848:ll_io_set_mirror()) f204e.sanity-flr: desiginated mirror: 1 00000080:00200000:2.0:1787826407.896500:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1787826407.896505:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1787826407.896508:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:2.0:1787826407.896510:0:69451:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [512, 18446744073709551615] 00000008:00010000:2.0:1787826407.896517:0:69451:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:2.0:1787826407.896519:0:69451:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322030, osc lock ffff907d830a3380, flags 0x0 00010000:00010000:2.0:1787826407.896528:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1787826407.896549:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (2097152 104857599) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1787826407.896559:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1787826407.896567:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 6/2,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1787826407.896573:0:69451:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1787826407.896583:0:69451:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x209:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826407.896588:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_release():245 00000020:00010000:2.0:1787826407.896591:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826407.896594:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322030 (ffff907d884a71b8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1787826407.896598:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 4/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1787826407.896634:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000020:00010000:2.0:1787826407.896644:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98af27b8 (ffff907d884a7d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826407.896647:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322030 (ffff907d884a7d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826407.896654:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826407.896658:0:69451:0:(file.c:5262:ll_file_seek()) COMPLETED file f204e.sanity-flr:[0x200000404:0x209:0x0], offset: 2097152, type: SEEK_DATA, rc = -6 00000080:00200008:2.0:1787826407.896665:0:69451:0:(file.c:5224:ll_file_seek()) START file f204e.sanity-flr:[0x200000404:0x209:0x0], offset: 0, type: (null) 00000080:00200000:2.0:1787826407.896670:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:2.0:1787826407.896674:0:69451:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826407.896676:0:69451:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1787826407.896678:0:69451:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x209:0x0] 00000008:00010000:2.0:1787826407.896684:0:69451:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1787826407.896686:0:69451:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d96322030, osc lock ffff907d830a3380, flags 0x400000400001002 00010000:00010000:2.0:1787826407.896694:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1787826407.896703:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 104857599) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00000020:00200000:2.0:1787826407.896711:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1787826407.896714:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1787826407.896720:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00000020:00200000:2.0:1787826407.896729:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1787826407.896734:0:69451:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x209:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1787826407.896736:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5792c70 (ffff907d884a7d88/0) at cl_lock_release():245 00000020:00010000:2.0:1787826407.896738:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5792c70 (ffff907d884a7d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1787826407.896741:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d96322030 (ffff907d884a71b8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1787826407.896743:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5792c70 (ffff907d884a7d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1787826407.896745:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d96322030 (ffff907d884a7d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1787826407.896751:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1787826407.896757:0:69451:0:(file.c:5262:ll_file_seek()) COMPLETED file f204e.sanity-flr:[0x200000404:0x209:0x0], offset: 0, type: (null), rc = 2097152 00000080:00200080:2.0:1787826407.896763:0:69451:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8) cmd=400866b1 arg=0 00000080:00200080:3.0:1787826407.912233:0:69451:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8) cmd=400866b1 arg=2 00000080:00200008:3.0:1787826407.912254:0:69451:0:(file.c:6313:ll_getattr_dentry()) START file f204e.sanity-flr:[0x200000404:0x209:0x0](ffff907d933121c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1787826407.912263:0:69451:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8),name=f204e.sanity-flr 00000002:00010000:3.0:1787826407.912284:0:69451:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x209:0x0]) in obj [0x200000404:0x209:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826407.912303:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1787826407.912319:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1787826407.912338:0:69451:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x209:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826407.912347:0:69451:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) 00000080:00010000:3.0:1787826407.912351:0:69451:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) for lock 0xe5f736940357bb21 00000080:00010000:3.0:1787826407.912359:0:69451:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357bb21 from it ffff9dd142bd7cd8 00010000:00010000:3.0:1787826407.912363:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1787826407.912373:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1787826407.912399:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:3.0:1787826407.912411:0:69451:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:3.0:1787826407.912414:0:69451:0:(lov_io.c:750:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:3.0:1787826407.912418:0:69451:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000404:0x209:0x0] 00000008:00010000:3.0:1787826407.912437:0:69451:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1787826407.912441:0:69451:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d98a1ceb0, osc lock ffff907d963b9c08, flags 0x400000400001002 00010000:00010000:3.0:1787826407.912457:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:3.0:1787826407.912468:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 104857599) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00000020:00200000:3.0:1787826407.912481:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1787826407.912484:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 5/1,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 9 lru_type: 0 00010000:00010000:3.0:1787826407.912492:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 4/0,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 0 00000020:00200000:3.0:1787826407.912507:0:69451:0:(cl_io.c:1853:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1787826407.912517:0:69451:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x209:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:3.0:1787826407.912521:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907da5791638 (ffff907d884a7098/0) at cl_lock_release():245 00000020:00010000:3.0:1787826407.912524:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da5791638 (ffff907d884a7098/0) at cl_lock_cancel():142 00000020:00010000:3.0:1787826407.912527:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98a1ceb0 (ffff907d884a76c8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1787826407.912530:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da5791638 (ffff907d884a7098/0) at cl_lock_fini():72 00000020:00010000:3.0:1787826407.912544:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98a1ceb0 (ffff907d884a7098/1) at cl_lock_fini():72 00000080:00200000:3.0:1787826407.912556:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:3.0:1787826407.912584:0:69451:0:(llite_lib.c:2427:ll_setattr_raw()) START file f204e.sanity-flr:[0x200000404:0x209:0x0](ffff907d933121c8) current size 2097152, valid attrs 2068, mode 0, uid -1976721614, gid -1, new size 2097152, atime 1787826407.000000000, mtime 1787826407.000000000, ctime 1787826407.000000000 00010000:00010000:3.0:1787826407.912635:0:69451:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1787826407.912652:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1787826407.912694:0:69451:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1787826407.912701:0:69451:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787826407.912704:0:69451:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883b600/0xe5f736940357bb21 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc108d7 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 6 lru_type: 0 00000020:00100000:3.0:1787826407.912737:0:69451:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1787826407.912750:0:69451:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83524780 x1874670011472896/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826407.912779:0:69451:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011472896:192.168.203.104@tcp:36:lfs.0 00000100:00100000:3.0:1787826407.912876:0:69451:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da0397240 going to sleep for 13 seconds 00000100:00100000:3.0:1787826407.917194:0:69451:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011472896:192.168.203.104@tcp:36:lfs.0 00000080:00200000:3.0:1787826407.917240:0:69451:0:(file.c:1848:ll_io_set_mirror()) f204e.sanity-flr: desiginated mirror: 2 00000080:00200000:3.0:1787826407.917247:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:3.0:1787826407.917265:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x11 00020000:00200000:3.0:1787826407.917277:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[3] flags 0x11 00020000:00200000:3.0:1787826407.917283:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[4] flags 0x40 00000080:00200000:3.0:1787826407.917287:0:69451:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:3.0:1787826407.917301:0:69451:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787826407.917305:0:69451:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da0397270, osc lock ffff907d963b9c08, flags 0x0 00000008:00010000:3.0:1787826407.917309:0:69451:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1787826407.917310:0:69451:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907da03972b0, osc lock ffff907d963b94b8, flags 0x0 00010000:00010000:3.0:1787826407.917371:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000072158d5f/0xe5f736940357bb2f lrc: 3/0,1 mode: --/PW res: [0x240000400:0x32a:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.917383:0:69451:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000072158d5f/0xe5f736940357bb2f lrc: 3/0,1 mode: --/PW res: [0x240000400:0x32a:0x0].0x0 rrc: 2 type: EXT [0->67108863] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.917395:0:69451:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000072158d5f/0xe5f736940357bb2f lrc: 3/0,1 mode: --/PW res: [0x240000400:0x32a:0x0].0x0 rrc: 2 type: EXT [0->67108863] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826407.917414:0:69451:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011473024:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826407.917483:0:69451:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d98247cc0 going to sleep for 11 seconds 00000100:00100000:3.0:1787826407.920326:0:69451:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011473024:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826407.920364:0:69451:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000072158d5f/0xe5f736940357bb2f lrc: 4/0,1 mode: --/PW res: [0x240000400:0x32a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc108fa expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.920388:0:69451:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000072158d5f/0xe5f736940357bb2f lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x32a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc108fa expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.920400:0:69451:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000072158d5f/0xe5f736940357bb2f lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x32a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc108fa expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.920415:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000072158d5f/0xe5f736940357bb2f lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x32a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc108fa expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:3.0:1787826407.920428:0:69451:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000072158d5f/0xe5f736940357bb2f lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x32a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc108fa expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.920441:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000072158d5f/0xe5f736940357bb2f lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x32a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108fa expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.920449:0:69451:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000072158d5f/0xe5f736940357bb2f lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x32a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108fa expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.920519:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000006df4f474/0xe5f736940357bb36 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30f:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.920527:0:69451:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000006df4f474/0xe5f736940357bb36 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30f:0x0].0x0 rrc: 2 type: EXT [67108864->134217727] (req 67108864->134217727) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826407.920551:0:69451:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000006df4f474/0xe5f736940357bb36 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x30f:0x0].0x0 rrc: 2 type: EXT [67108864->134217727] (req 67108864->134217727) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826407.920585:0:69451:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011473152:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826407.926149:0:69451:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d98247cc0 going to sleep for 11 seconds 00000100:00100000:0.0:1787826407.942595:0:69451:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525a40 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011473152:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826407.942652:0:69451:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000006df4f474/0xe5f736940357bb36 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x30f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10901 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.942672:0:69451:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000006df4f474/0xe5f736940357bb36 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10901 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.942684:0:69451:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000006df4f474/0xe5f736940357bb36 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10901 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.942697:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000006df4f474/0xe5f736940357bb36 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x30f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10901 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:0.0:1787826407.942710:0:69451:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000006df4f474/0xe5f736940357bb36 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x30f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10901 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.942721:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000006df4f474/0xe5f736940357bb36 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x30f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10901 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.942728:0:69451:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000006df4f474/0xe5f736940357bb36 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10901 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826407.942995:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011473280:192.168.203.104@tcp:10:lfs.0 00000100:00100000:1.0:1787826407.943101:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1787826407.943120:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011473408:192.168.203.104@tcp:10:lfs.0 00000100:00100000:1.0:1787826407.949245:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011473408:192.168.203.104@tcp:10:lfs.0 00000100:00100000:1.0:1787826407.949283:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011473280:192.168.203.104@tcp:10:lfs.0 00000080:00200000:0.0:1787826407.950384:0:69451:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000404:0x209:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:0.0:1787826407.950402:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d904b1128 (ffff907d884a7098/0) at cl_lock_release():245 00000020:00010000:0.0:1787826407.950407:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d904b1128 (ffff907d884a7098/0) at cl_lock_cancel():142 00000020:00010000:0.0:1787826407.950414:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da0397270 (ffff907d884a76c8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1787826407.950425:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000072158d5f/0xe5f736940357bb2f lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x32a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc108fa expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.950441:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000072158d5f/0xe5f736940357bb2f lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x32a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108fa expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:0.0:1787826407.950464:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907da03972b0 (ffff907d884a7128/1) at cl_lock_cancel():142 00010000:00010000:0.0:1787826407.950469:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000006df4f474/0xe5f736940357bb36 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x30f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7918213f4fc10901 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826407.950477:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000006df4f474/0xe5f736940357bb36 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x30f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10901 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:0.0:1787826407.950487:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d904b1128 (ffff907d884a7098/0) at cl_lock_fini():72 00000020:00010000:0.0:1787826407.950490:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da0397270 (ffff907d884a7098/1) at cl_lock_fini():72 00000020:00010000:0.0:1787826407.950498:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907da03972b0 (ffff907d884a7098/1) at cl_lock_fini():72 00000080:00200000:0.0:1787826407.950519:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826407.950723:0:69451:0:(llite_lib.c:2686:ll_setattr_raw()) COMPLETED file f204e.sanity-flr:[0x200000404:0x209:0x0](ffff907d933121c8) current size 2097152, valid attrs 10002168, mode 0, uid -1976721614, gid -1, new size 2097152, atime 1787826407.000000000, mtime 1787826407.000000000, ctime 1787826407.000000000, rc 0 00000080:00200080:1.0:1787826407.950744:0:69451:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8) cmd=400866b1 arg=0 00000080:00200008:1.0:1787826407.950774:0:69451:0:(llite_lib.c:2427:ll_setattr_raw()) START file f204e.sanity-flr:[0x200000404:0x209:0x0](ffff907d933121c8) current size 2097152, valid attrs 101f0, mode 9dd1, uid -1786542832, gid -28547, new size 18446621467747623368, atime 1787826381.000000000, mtime 1787826394.000000000, ctime 1787826407.000000000 00000020:00100000:1.0:1787826407.950842:0:69451:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1787826407.950855:0:69451:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a1e00 x1874670011473664/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826407.950880:0:69451:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011473664:192.168.203.104@tcp:36:lfs.0 00000100:00100000:1.0:1787826407.951065:0:69451:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7d780 going to sleep for 13 seconds 00000100:00100000:1.0:1787826407.959932:0:69451:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011473664:192.168.203.104@tcp:36:lfs.0 00000080:00200000:1.0:1787826407.959995:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 6 restore needed 0 00020000:00200000:1.0:1787826407.960012:0:69451:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826407.960016:0:69451:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826407.960026:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1787826407.960044:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:1.0:1787826407.960048:0:69451:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1787826407.960067:0:69451:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1787826407.960072:0:69451:0:(osc_lock.c:1320:osc_lock_init()) ### lock ffff907d88c323b0, osc lock ffff907d962f49c0, flags 0x0 00010000:00010000:1.0:1787826407.960094:0:69451:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d91e6e400 type 11 mode 2 res 9663677440/809 (0 104857599) 00010000:00010000:1.0:1787826407.960166:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 3/0,1 mode: --/PW res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.961217:0:69451:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 3/0,1 mode: --/PW res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->104857599] (req 0->104857599) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.961235:0:69451:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 3/0,1 mode: --/PW res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->104857599] (req 0->104857599) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826407.961270:0:69451:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011473792:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826407.966477:0:69451:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7d780 going to sleep for 11 seconds 00000100:00100000:3.0:1787826407.975479:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826407.975557:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025522816 00000100:00100000:3.0:1787826407.975568:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025522816 00000100:00100000:3.0:1787826407.975658:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7166 00000100:00100000:3.0:1787826407.975670:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d83524f00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:6935:x1874670025522816:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826407.975693:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 0 00010000:00010000:3.0:1787826407.975846:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826407.975854:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d83524f00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:6935:x1874670025522816:12345-192.168.203.104@tcp:104: Request processed in 187us (389us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826407.975862:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7166 00000100:00100000:2.0:1787826407.978430:0:69451:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011473792:192.168.203.104@tcp:101:lfs.0 00010000:00010000:2.0:1787826407.978473:0:69451:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 4/0,1 mode: --/PW res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->104857599] (req 0->104857599) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.978492:0:69451:0:(ldlm_request.c:298:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 4/0,1 mode: --/PW res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->104857599] (req 0->104857599) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.978502:0:69451:0:(ldlm_request.c:327:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 4/0,1 mode: --/PW res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->104857599] (req 0->104857599) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.982053:0:61297:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826407.982078:0:61297:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800420000000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1787826407.982097:0:61297:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a600 already unused, calling callback (ffffffffc118e410) 00010000:00010000:1.0:1787826407.982229:0:61297:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 3/0,0 mode: PR/PR res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800428400000000 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1787826407.982260:0:61297:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1787826407.982279:0:61297:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1787826407.982465:0:61297:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 2/0,0 mode: --/PR res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1787826407.982475:0:61297:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826407.982527:0:61297:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 1/0,0 mode: --/PR res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1787826407.982551:0:61297:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000c925444c/0xe5f736940357bb28 lrc: 0/0,0 mode: --/PR res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7918213f4fc108de expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 10 lru_type: 0 00010000:00010000:1.0:1787826407.982577:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826407.982579:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826407.982581:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826407.982689:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011473920:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1787826407.985388:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826407.986023:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025522944 00000100:00100000:2.0:1787826407.986038:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025522944 00000100:00100000:2.0:1787826407.986114:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7167 00000100:00100000:2.0:1787826407.986131:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d835243c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:5932:x1874670025522944:12345-192.168.203.104@tcp:105: 00010000:00010000:2.0:1787826407.986152:0:40338:0:(ldlm_lockd.c:2518:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 5/0,1 mode: --/PW res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->104857599] (req 0->104857599) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.986165:0:40338:0:(ldlm_lockd.c:1968:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 5/0,1 mode: --/PW res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->104857599] (req 0->104857599) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.986175:0:40338:0:(ldlm_lockd.c:2045:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 5/0,1 mode: --/PW res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.986185:0:40338:0:(ldlm_lock.c:715:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x10080000020000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.986196:0:40338:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x10080000020000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.986205:0:40338:0:(ldlm_lockd.c:2073:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x80000020000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826407.986238:0:40338:0:(ldlm_lockd.c:2084:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff907d83f0bc00) 00010000:00010000:1.0:1787826407.986298:0:69451:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.986319:0:69451:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x20000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826407.986327:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d835243c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:5932:x1874670025522944:12345-192.168.203.104@tcp:105: Request processed in 201us (951us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826407.986341:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7167 00010000:00010000:1.0:1787826407.986362:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x10000000020000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1787826407.986381:0:69451:0:(osc_lock.c:192:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x10020000020000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.986398:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826407.986407:0:69451:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826407.988296:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1787826407.988389:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011474048:192.168.203.104@tcp:2:lfs.0 00000100:00100000:1.0:1787826407.999567:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011473920:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826408.000902:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011474048:192.168.203.104@tcp:2:lfs.0 00000020:00010000:1.0:1787826408.001400:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff907d98af2f38 (ffff907d884a7368/0) at cl_lock_release():245 00000020:00010000:1.0:1787826408.001407:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d98af2f38 (ffff907d884a7368/0) at cl_lock_cancel():142 00000020:00010000:1.0:1787826408.001414:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff907d88c323b0 (ffff907da5608ea8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1787826408.001428:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x810020000020000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826408.001448:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:1.0:1787826408.001468:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d98af2f38 (ffff907d884a7368/0) at cl_lock_fini():72 00000020:00010000:1.0:1787826408.001471:0:69451:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff907d88c323b0 (ffff907d884a7368/1) at cl_lock_fini():72 00000080:00200000:1.0:1787826408.001495:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1787826408.001511:0:69451:0:(llite_lib.c:2686:ll_setattr_raw()) COMPLETED file f204e.sanity-flr:[0x200000404:0x209:0x0](ffff907d933121c8) current size 2097152, valid attrs 100101f0, mode 9dd1, uid -1786542832, gid -28547, new size 18446621467747623368, atime 1787826381.000000000, mtime 1787826394.000000000, ctime 1787826407.000000000, rc 0 00000080:00200080:1.0:1787826408.001549:0:69451:0:(file.c:4655:ll_file_ioctl()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8) cmd=c00c66f3 arg=15f02a0 00010000:00010000:1.0:1787826408.001641:0:69451:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ac00/0xe5f736940357bb13 lrc: 2/0,0 mode: PW/PW res: [0x200000404:0x209:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x81008000000000 nid: local remote: 0x7918213f4fc108ad expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826408.001663:0:69451:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ac00/0xe5f736940357bb13 lrc: 0/0,0 mode: --/PW res: [0x200000404:0x209:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x85809400000000 nid: local remote: 0x7918213f4fc108ad expref: -99 pid: 69451 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826408.001689:0:69451:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d9114bc00 x1874670011471232/t8589943035(8589943035) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826425 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1787826408.001702:0:69451:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1787826408.001707:0:69451:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a21c0 x1874670011474304/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 504/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1787826408.001735:0:69451:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011474304:192.168.203.104@tcp:35:lfs.0 00000100:00100000:1.0:1787826408.001859:0:69451:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7dc00 going to sleep for 11 seconds 00000100:00100000:2.0:1787826408.006883:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826408.007803:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025523072 00000100:00100000:3.0:1787826408.007820:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025523072 00000100:00100000:3.0:1787826408.007897:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7168 00000100:00100000:3.0:1787826408.007907:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91149a40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:33311:x1874670025523072:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826408.007931:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1787826408.008039:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826408.008046:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91149a40 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:6f891bcb-15cd-41e0-96e2-db843619c1be+5:33311:x1874670025523072:12345-192.168.203.104@tcp:104: Request processed in 141us (1175us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826408.008054:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7168 00010000:00010000:0.0:1787826408.008266:0:47852:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826408.008275:0:47852:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1787826408.008286:0:47852:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826408.008293:0:47852:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1787826408.008309:0:47852:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1787826408.008319:0:47852:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 3/0,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:0.0:1787826408.008358:0:47852:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x209:0x0]: losing layout lock 00010000:00010000:0.0:1787826408.008418:0:47852:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 2/0,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1787826408.008425:0:47852:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826408.008454:0:47852:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 1/0,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1787826408.008463:0:47852:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb1a lrc: 0/0,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7918213f4fc108c9 expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1787826408.008481:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826408.008482:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826408.008484:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826408.008492:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1787826408.008508:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011474432:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1787826408.021968:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011474432:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826408.041352:0:69451:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589943040 gen 1 00000100:00100000:1.0:1787826408.041400:0:69451:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011474304:192.168.203.104@tcp:35:lfs.0 00000002:00100000:1.0:1787826408.041436:0:69451:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d9114bc00 x1874670011471232/t8589943035(8589943035) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826425 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:1.0:1787826408.041479:0:69451:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 31 res 8589935620/521 (0 0) 00000080:00010000:1.0:1787826408.041493:0:69451:0:(file.c:7076:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000404:0x209:0x0](ffff907d933121c8) 00000002:00010000:1.0:1787826408.041507:0:69451:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x209:0x0]) in obj [0x200000404:0x209:0x0], intent: layout flags 0 00010000:00010000:1.0:1787826408.041572:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69451 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826408.041593:0:69451:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69451 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826408.041605:0:69451:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69451 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826408.041669:0:69451:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011474560:192.168.203.104@tcp:101:lfs.0 00000100:00100000:3.0:1787826408.043506:0:69451:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7dc00 going to sleep for 13 seconds 00000100:00100000:3.0:1787826408.058576:0:69451:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011474560:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826408.058685:0:69451:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1091d expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.058704:0:69451:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 4/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1091d expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826408.058723:0:69451:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x400 disposition=0x1, status=0 req@ffff907d908a21c0 x1874670011474560/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 376/896 e 0 to 0 dl 1787826426 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1787826408.058737:0:69451:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1091d expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1787826408.058785:0:69451:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1091d expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.058802:0:69451:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 16 res 8589935620/521 (0 0) 00000080:00200000:3.0:1787826408.058810:0:69451:0:(namei.c:952:ll_intent_lock()) intent lock 1024 on i1 [0x200000404:0x209:0x0] suppgids 0 0: rc 0 00000080:00010000:3.0:1787826408.058838:0:69451:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) for lock 0xe5f736940357bb44 00000080:00010000:3.0:1787826408.058849:0:69451:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357bb44 from it ffff9dd142bd7bc8 00010000:00010000:3.0:1787826408.058855:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1091d expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.058867:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1091d expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826408.064643:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1091d expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826408.064674:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1091d expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00010000:2.0:1787826408.064685:0:69451:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x209:0x0](000000003c30eba3) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1091d expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826408.064747:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1091d expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826408.064758:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1091d expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1787826408.064780:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1787826408.064791:0:69451:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826408.064794:0:69451:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1787826408.064801:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1787826408.064823:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:2.0:1787826408.064845:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1787826408.064861:0:69451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1091d expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826408.064869:0:69451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 2/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1091d expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:2.0:1787826408.064876:0:69451:0:(file.c:6959:ll_layout_lock_set()) ### file [0x200000404:0x209:0x0](000000003c30eba3) being reconfigured ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1091d expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1787826408.064899:0:69451:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 1/0, layout version 8 restore needed 0 00020000:00200000:2.0:1787826408.064904:0:69451:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1787826408.064906:0:69451:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1787826408.064909:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1787826408.064913:0:69451:0:(lov_io.c:1253:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:2.0:1787826408.064919:0:69451:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:2.0:1787826408.064943:0:69451:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 2097152 blocks 4096 [cma]time: 1787826394 1787826394 0 by oinfo size 2097152 blocks 4096 [cma]time 1787826407 1787826394 1787826381 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x329:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x800020000020000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:2.0:1787826408.064972:0:69451:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 2097152 blocks 4096 [cma]time: 1787826407 1787826407 0 by oinfo size 2097152 blocks 4096 [cma]time 1787826407 1787826407 0 ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000072158d5f/0xe5f736940357bb2f lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x32a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc108fa expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:2.0:1787826408.064990:0:69451:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 0 blocks 0 [cma]time 1787826407 1787826407 0 ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000006df4f474/0xe5f736940357bb36 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x30f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7918213f4fc10901 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1787826408.065051:0:69451:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x209:0x0]: layout version change: 4294967294 -> 8 00010000:00010000:2.0:1787826408.065057:0:69451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 3/1,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc1091d expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826408.065066:0:69451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc1091d expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200008:2.0:1787826408.065106:0:69451:0:(file.c:479:ll_file_release()) START file f204e.sanity-flr:[0x200000404:0x209:0x0](ffff907d933121c8), flags 154002 00000002:00100000:2.0:1787826408.065167:0:69451:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d9114a580 x1874670011470848/t8589943034(8589943034) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826425 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1787826408.065180:0:69451:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1787826408.065186:0:69451:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d911483c0 x1874670011474816/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1787826408.065210:0:69451:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011474816:192.168.203.104@tcp:35:lfs.0 00000100:00100000:2.0:1787826408.065330:0:69451:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d87248c00 going to sleep for 11 seconds 00000100:00100000:0.0:1787826408.068655:0:69451:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69451:1874670011474816:192.168.203.104@tcp:35:lfs.0 00000002:00100000:0.0:1787826408.068701:0:69451:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff907d9114a580 x1874670011470848/t8589943034(8589943034) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1787826425 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826408.097825:0:69452:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826408.097867:0:69452:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826408.097891:0:69452:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:1.0:1787826408.097912:0:69452:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00000080:00200000:1.0:1787826408.097929:0:69452:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826408.097939:0:69452:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826408.097943:0:69452:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:1.0:1787826408.097952:0:69452:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142b47bf8 00010000:00010000:1.0:1787826408.097956:0:69452:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 31 lru_type: 1 00010000:00010000:1.0:1787826408.097967:0:69452:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00000080:00200000:1.0:1787826408.097986:0:69452:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826408.097995:0:69452:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=27 00000080:00200000:1.0:1787826408.098002:0:69452:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:1.0:1787826408.098018:0:69452:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), file ffff907d99bc2f00, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1787826408.098027:0:69452:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=open 00000002:00010000:1.0:1787826408.098045:0:69452:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x208:0x0], intent: open flags 0304001 00010000:00010000:1.0:1787826408.098247:0:69452:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb4b lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 69452 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826408.098258:0:69452:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb4b lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69452 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1787826408.098273:0:69452:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1787826408.098289:0:69452:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a1e00 x1874670011475072/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1787826408.098300:0:69452:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb4b lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69452 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826408.098396:0:69452:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69452:1874670011475072:192.168.203.104@tcp:101:lfs.0 00000100:00100000:1.0:1787826408.098917:0:69452:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7d300 going to sleep for 13 seconds 00000100:00100000:3.0:1787826408.104135:0:69452:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69452:1874670011475072:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826408.104167:0:69452:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb4b lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69452 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.104183:0:69452:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb4b lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 69452 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.104190:0:69452:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb4b lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 69452 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.104198:0:69452:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb4b lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 69452 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.104209:0:69452:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb4b lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 69452 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.104216:0:69452:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0b400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826408.104221:0:69452:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb4b lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 69452 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.104229:0:69452:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb4b lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 69452 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.104238:0:69452:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb4b lrc: 5/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 69452 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.104246:0:69452:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826408.104255:0:69452:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb4b lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 69452 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.104262:0:69452:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb4b lrc: 0/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 69452 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826408.104291:0:69452:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d908a1e00 x1874670011475072/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826426 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826408.104307:0:69452:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x208:0x0] suppgids 0 -1: rc -20 00000080:00200000:3.0:1787826408.104484:0:69452:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826408.104509:0:69452:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826408.104528:0:69452:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00010000:00010000:3.0:1787826408.104552:0:69452:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00000080:00200000:3.0:1787826408.104560:0:69452:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826408.104566:0:69452:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826408.104570:0:69452:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:3.0:1787826408.104577:0:69452:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142b47bf8 00010000:00010000:3.0:1787826408.104581:0:69452:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 32 lru_type: 1 00010000:00010000:3.0:1787826408.104589:0:69452:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00000080:00200000:3.0:1787826408.104976:0:69452:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826408.104988:0:69452:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=267 00000080:00200000:3.0:1787826408.104997:0:69452:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 44 00000080:00200008:3.0:1787826408.105007:0:69452:0:(file.c:1090:ll_file_open()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00000080:00200080:3.0:1787826408.105065:0:69452:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8) cmd=c0086915 arg=256d3d0 00000100:00100000:3.0:1787826408.105144:0:69452:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83526580 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69452:1874670011475200:192.168.203.104@tcp:34:lfs.0 00000100:00100000:3.0:1787826408.105253:0:69452:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d98247d80 going to sleep for 13 seconds 00000100:00100000:3.0:1787826408.112046:0:69452:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83526580 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69452:1874670011475200:192.168.203.104@tcp:34:lfs.0 00000080:00200008:3.0:1787826408.112992:0:69452:0:(file.c:479:ll_file_release()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00010000:00010000:3.0:1787826408.113036:0:69452:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x208:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc107cd expref: -99 pid: 69437 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200008:0.0:1787826408.147955:0:69454:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1787826408.147968:0:69454:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826408.147991:0:69454:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826408.148012:0:69454:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00010000:00010000:0.0:1787826408.148029:0:69454:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00000080:00200000:0.0:1787826408.148038:0:69454:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826408.148047:0:69454:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826408.148051:0:69454:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:0.0:1787826408.148060:0:69454:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142b47cb8 00010000:00010000:0.0:1787826408.148064:0:69454:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 33 lru_type: 1 00010000:00010000:0.0:1787826408.148073:0:69454:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00000080:00200000:0.0:1787826408.148193:0:69454:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826408.148200:0:69454:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826408.148207:0:69454:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00010000:00010000:0.0:1787826408.148216:0:69454:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00000080:00200000:0.0:1787826408.148228:0:69454:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826408.148232:0:69454:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826408.148235:0:69454:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:0.0:1787826408.148238:0:69454:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142b47bb0 00010000:00010000:0.0:1787826408.148242:0:69454:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 34 lru_type: 1 00010000:00010000:0.0:1787826408.148250:0:69454:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00000080:00200000:0.0:1787826408.148258:0:69454:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:0.0:1787826408.148265:0:69454:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:0.0:1787826408.148282:0:69454:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7ffc6086366c 00000080:00200008:0.0:1787826408.148296:0:69454:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:2.0:1787826408.155320:0:69457:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826408.155360:0:69457:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826408.155385:0:69457:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00010000:00010000:2.0:1787826408.155421:0:69457:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00000080:00200000:2.0:1787826408.155429:0:69457:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826408.155440:0:69457:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826408.155444:0:69457:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:2.0:1787826408.155452:0:69457:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142b6fbf8 00010000:00010000:2.0:1787826408.155456:0:69457:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 35 lru_type: 1 00010000:00010000:2.0:1787826408.155465:0:69457:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00000080:00200000:2.0:1787826408.155484:0:69457:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826408.155492:0:69457:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=27 00000080:00200000:2.0:1787826408.155500:0:69457:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:2.0:1787826408.155527:0:69457:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), file ffff907d87186f00, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1787826408.155544:0:69457:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=open 00000002:00010000:2.0:1787826408.155563:0:69457:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x208:0x0], intent: open flags 0304001 00010000:00010000:2.0:1787826408.155659:0:69457:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ba00/0xe5f736940357bb52 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 69457 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826408.155668:0:69457:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ba00/0xe5f736940357bb52 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69457 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826408.155682:0:69457:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826408.155697:0:69457:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114b480 x1874670011475456/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826408.155708:0:69457:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ba00/0xe5f736940357bb52 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69457 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826408.155737:0:69457:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69457:1874670011475456:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826408.155998:0:69457:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d87248c00 going to sleep for 13 seconds 00000100:00100000:1.0:1787826408.159499:0:69457:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69457:1874670011475456:192.168.203.104@tcp:101:lfs.0 00010000:00010000:1.0:1787826408.159713:0:69457:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ba00/0xe5f736940357bb52 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69457 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826408.159731:0:69457:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ba00/0xe5f736940357bb52 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 69457 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826408.159739:0:69457:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ba00/0xe5f736940357bb52 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 69457 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826408.159746:0:69457:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ba00/0xe5f736940357bb52 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 69457 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826408.159756:0:69457:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ba00/0xe5f736940357bb52 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 69457 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826408.159764:0:69457:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8646ba00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:1.0:1787826408.159769:0:69457:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ba00/0xe5f736940357bb52 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 69457 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826408.159777:0:69457:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ba00/0xe5f736940357bb52 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 69457 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826408.159784:0:69457:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ba00/0xe5f736940357bb52 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 69457 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826408.159792:0:69457:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1787826408.159802:0:69457:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ba00/0xe5f736940357bb52 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 69457 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826408.159809:0:69457:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646ba00/0xe5f736940357bb52 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 69457 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826408.159833:0:69457:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d9114b480 x1874670011475456/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826426 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1787826408.159849:0:69457:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x208:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1787826408.163312:0:69457:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826408.163352:0:69457:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826408.163377:0:69457:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00010000:00010000:1.0:1787826408.164298:0:69457:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00000080:00200000:1.0:1787826408.164313:0:69457:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826408.164327:0:69457:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826408.164332:0:69457:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:1.0:1787826408.164342:0:69457:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142b6fbf8 00010000:00010000:1.0:1787826408.164346:0:69457:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 36 lru_type: 1 00010000:00010000:1.0:1787826408.164357:0:69457:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00000080:00200000:1.0:1787826408.164380:0:69457:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826408.165219:0:69457:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=267 00000080:00200000:1.0:1787826408.165237:0:69457:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 44 00000080:00200008:1.0:1787826408.165253:0:69457:0:(file.c:1090:ll_file_open()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00000080:00200080:1.0:1787826408.165329:0:69457:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8) cmd=c1406916 arg=1871a20 00000100:00100000:1.0:1787826408.165458:0:69457:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69457:1874670011475584:192.168.203.104@tcp:34:lfs.0 00000100:00100000:1.0:1787826408.166917:0:69457:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88c7d300 going to sleep for 13 seconds 00000100:00100000:1.0:1787826408.181579:0:69457:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69457:1874670011475584:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826408.181823:0:69457:0:(file.c:479:ll_file_release()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00010000:00010000:1.0:1787826408.181864:0:69457:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x208:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc107cd expref: -99 pid: 69437 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200008:3.0:1787826408.231444:0:69461:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1787826408.231457:0:69461:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826408.231484:0:69461:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826408.231506:0:69461:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00010000:00010000:3.0:1787826408.231524:0:69461:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00000080:00200000:3.0:1787826408.231544:0:69461:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826408.231553:0:69461:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826408.231557:0:69461:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:3.0:1787826408.231565:0:69461:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142bdfcb8 00010000:00010000:3.0:1787826408.231569:0:69461:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 37 lru_type: 1 00010000:00010000:3.0:1787826408.231579:0:69461:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00000080:00200000:3.0:1787826408.232053:0:69461:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826408.232066:0:69461:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826408.232077:0:69461:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00010000:00010000:3.0:1787826408.232088:0:69461:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00000080:00200000:3.0:1787826408.232095:0:69461:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826408.232099:0:69461:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826408.232102:0:69461:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:3.0:1787826408.232106:0:69461:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142bdfbb0 00010000:00010000:3.0:1787826408.232110:0:69461:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 38 lru_type: 1 00010000:00010000:3.0:1787826408.232118:0:69461:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00000080:00200000:3.0:1787826408.232128:0:69461:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:3.0:1787826408.232137:0:69461:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:3.0:1787826408.232168:0:69461:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7fffcd8c1d6c 00000080:00200008:3.0:1787826408.232186:0:69461:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:0.0:1787826408.233953:0:69465:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826408.233981:0:69465:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826408.234003:0:69465:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00010000:00010000:0.0:1787826408.234019:0:69465:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00000080:00200000:0.0:1787826408.234028:0:69465:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826408.234036:0:69465:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826408.234040:0:69465:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:0.0:1787826408.234047:0:69465:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142c0fbf8 00010000:00010000:0.0:1787826408.234052:0:69465:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 39 lru_type: 1 00010000:00010000:0.0:1787826408.234061:0:69465:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00000080:00200000:0.0:1787826408.234077:0:69465:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826408.234085:0:69465:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=27 00000080:00200000:0.0:1787826408.234093:0:69465:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:0.0:1787826408.234117:0:69465:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), file ffff907d85050d80, open_flags 18800, mode 0 opened 0 00000080:00200000:0.0:1787826408.234126:0:69465:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=open 00000002:00010000:0.0:1787826408.234142:0:69465:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x208:0x0], intent: open flags 0304001 00010000:00010000:0.0:1787826408.234211:0:69465:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb59 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 69465 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826408.234220:0:69465:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb59 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69465 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826408.234233:0:69465:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826408.234257:0:69465:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90847480 x1874670011475840/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826408.234268:0:69465:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb59 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69465 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826408.234295:0:69465:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69465:1874670011475840:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826408.234506:0:69465:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238600 going to sleep for 13 seconds 00000100:00100000:0.0:1787826408.239431:0:69465:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69465:1874670011475840:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826408.239462:0:69465:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb59 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69465 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826408.239481:0:69465:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb59 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 69465 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826408.239490:0:69465:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb59 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 69465 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826408.239498:0:69465:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb59 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 69465 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826408.239508:0:69465:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb59 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 69465 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826408.239515:0:69465:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826408.239520:0:69465:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb59 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 69465 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826408.239528:0:69465:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb59 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 69465 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826408.239546:0:69465:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb59 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 69465 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826408.239555:0:69465:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826408.239564:0:69465:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb59 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 69465 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826408.239572:0:69465:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb59 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 69465 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826408.239592:0:69465:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d90847480 x1874670011475840/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826426 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826408.239625:0:69465:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x208:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1787826408.239896:0:69465:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826408.239921:0:69465:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826408.239939:0:69465:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00010000:00010000:0.0:1787826408.239952:0:69465:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00000080:00200000:0.0:1787826408.239961:0:69465:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826408.239966:0:69465:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826408.239971:0:69465:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:0.0:1787826408.239977:0:69465:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142c0fbf8 00010000:00010000:0.0:1787826408.239980:0:69465:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 40 lru_type: 1 00010000:00010000:0.0:1787826408.239989:0:69465:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00000080:00200000:0.0:1787826408.240004:0:69465:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826408.240012:0:69465:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=267 00000080:00200000:0.0:1787826408.240019:0:69465:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 44 00000080:00200008:0.0:1787826408.240027:0:69465:0:(file.c:1090:ll_file_open()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00000080:00200080:0.0:1787826408.241844:0:69465:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8) cmd=c1406916 arg=1e2aa20 00000100:00100000:0.0:1787826408.241955:0:69465:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69465:1874670011475968:192.168.203.104@tcp:34:lfs.0 00000100:00100000:0.0:1787826408.242221:0:69465:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238600 going to sleep for 13 seconds 00000100:00100000:0.0:1787826408.253835:0:69465:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69465:1874670011475968:192.168.203.104@tcp:34:lfs.0 00000080:00200008:0.0:1787826408.254054:0:69465:0:(file.c:479:ll_file_release()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00010000:00010000:0.0:1787826408.254094:0:69465:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x208:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc107cd expref: -99 pid: 69437 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200008:0.0:1787826408.282885:0:69468:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1787826408.282898:0:69468:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826408.282923:0:69468:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826408.282951:0:69468:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00010000:00010000:0.0:1787826408.282971:0:69468:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00000080:00200000:0.0:1787826408.282981:0:69468:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826408.282990:0:69468:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826408.282994:0:69468:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:0.0:1787826408.283003:0:69468:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142c17cb8 00010000:00010000:0.0:1787826408.283007:0:69468:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 41 lru_type: 1 00010000:00010000:0.0:1787826408.283017:0:69468:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00000080:00200000:0.0:1787826408.283333:0:69468:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826408.283342:0:69468:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826408.283351:0:69468:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00010000:00010000:0.0:1787826408.283365:0:69468:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00000080:00200000:0.0:1787826408.283372:0:69468:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826408.283376:0:69468:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826408.283379:0:69468:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:0.0:1787826408.283384:0:69468:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142c17bb0 00010000:00010000:0.0:1787826408.283387:0:69468:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 42 lru_type: 1 00010000:00010000:0.0:1787826408.283395:0:69468:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 43 lru_type: 1 00000080:00200000:0.0:1787826408.283405:0:69468:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:0.0:1787826408.283413:0:69468:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:0.0:1787826408.283433:0:69468:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7fff0f55764c 00000080:00200008:0.0:1787826408.283448:0:69468:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:2.0:1787826408.284893:0:69471:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826408.284916:0:69471:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826408.284931:0:69471:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 43 lru_type: 1 00010000:00010000:2.0:1787826408.284946:0:69471:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 43 lru_type: 1 00000080:00200000:2.0:1787826408.284954:0:69471:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826408.284961:0:69471:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826408.284965:0:69471:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:2.0:1787826408.284972:0:69471:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142c1fbf8 00010000:00010000:2.0:1787826408.284976:0:69471:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 43 lru_type: 1 00010000:00010000:2.0:1787826408.284985:0:69471:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 44 lru_type: 1 00000080:00200000:2.0:1787826408.284999:0:69471:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826408.285005:0:69471:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=27 00000080:00200000:2.0:1787826408.285011:0:69471:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:2.0:1787826408.285023:0:69471:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), file ffff907d87187800, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1787826408.285031:0:69471:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=open 00000002:00010000:2.0:1787826408.285047:0:69471:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x208:0x0], intent: open flags 0304001 00010000:00010000:2.0:1787826408.285128:0:69471:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bb60 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 69471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826408.285136:0:69471:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bb60 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69471 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826408.285148:0:69471:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826408.285163:0:69471:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d9114b480 x1874670011476224/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826408.285174:0:69471:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bb60 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69471 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826408.285198:0:69471:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69471:1874670011476224:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826408.287258:0:69471:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d87248900 going to sleep for 13 seconds 00000100:00100000:3.0:1787826408.290030:0:69471:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69471:1874670011476224:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826408.290061:0:69471:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bb60 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.290205:0:69471:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bb60 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 69471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.290219:0:69471:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bb60 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 69471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.290227:0:69471:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bb60 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 69471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.290237:0:69471:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bb60 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 69471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.290243:0:69471:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8646b800 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826408.290249:0:69471:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bb60 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 69471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.290256:0:69471:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bb60 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 69471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.290264:0:69471:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bb60 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 69471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.290273:0:69471:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826408.290282:0:69471:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bb60 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 69471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.290289:0:69471:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646b800/0xe5f736940357bb60 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 69471 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826408.290311:0:69471:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d9114b480 x1874670011476224/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826426 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826408.290421:0:69471:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x208:0x0] suppgids 0 -1: rc -20 00000080:00200000:3.0:1787826408.292458:0:69471:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826408.292494:0:69471:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826408.292517:0:69471:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 44 lru_type: 1 00010000:00010000:3.0:1787826408.292544:0:69471:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 44 lru_type: 1 00000080:00200000:3.0:1787826408.292553:0:69471:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826408.292563:0:69471:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826408.292568:0:69471:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:3.0:1787826408.292575:0:69471:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142c1fbf8 00010000:00010000:3.0:1787826408.292579:0:69471:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 44 lru_type: 1 00010000:00010000:3.0:1787826408.292590:0:69471:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 45 lru_type: 1 00000080:00200000:3.0:1787826408.292634:0:69471:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826408.292644:0:69471:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=267 00000080:00200000:3.0:1787826408.292653:0:69471:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 44 00000080:00200008:3.0:1787826408.292663:0:69471:0:(file.c:1090:ll_file_open()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00000080:00200080:3.0:1787826408.292710:0:69471:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8) cmd=c1406916 arg=2024a20 00000100:00100000:3.0:1787826408.292796:0:69471:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83526580 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69471:1874670011476352:192.168.203.104@tcp:34:lfs.0 00000100:00100000:3.0:1787826408.292922:0:69471:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d98247d80 going to sleep for 13 seconds 00000100:00100000:1.0:1787826408.297861:0:69471:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83526580 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69471:1874670011476352:192.168.203.104@tcp:34:lfs.0 00000080:00200008:1.0:1787826408.297943:0:69471:0:(file.c:479:ll_file_release()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00010000:00010000:1.0:1787826408.297983:0:69471:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x208:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc107cd expref: -99 pid: 69437 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1787826408.310823:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:0.0:1787826408.369735:0:69475:0:(file.c:6313:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1787826408.369748:0:69475:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826408.369776:0:69475:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826408.369801:0:69475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 45 lru_type: 1 00010000:00010000:0.0:1787826408.369820:0:69475:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 45 lru_type: 1 00000080:00200000:0.0:1787826408.369829:0:69475:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826408.369840:0:69475:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826408.369844:0:69475:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:0.0:1787826408.369854:0:69475:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142c07cb8 00010000:00010000:0.0:1787826408.369858:0:69475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 45 lru_type: 1 00010000:00010000:0.0:1787826408.369868:0:69475:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 46 lru_type: 1 00000080:00200000:0.0:1787826408.370001:0:69475:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826408.370010:0:69475:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826408.370018:0:69475:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 46 lru_type: 1 00010000:00010000:0.0:1787826408.370028:0:69475:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 46 lru_type: 1 00000080:00200000:0.0:1787826408.370036:0:69475:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826408.370040:0:69475:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826408.370042:0:69475:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:0.0:1787826408.370046:0:69475:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142c07bb0 00010000:00010000:0.0:1787826408.370048:0:69475:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 46 lru_type: 1 00010000:00010000:0.0:1787826408.370056:0:69475:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 47 lru_type: 1 00000080:00200000:0.0:1787826408.370064:0:69475:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 44 00000080:00200008:0.0:1787826408.370073:0:69475:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200080:0.0:1787826408.370092:0:69475:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148) cmd=800866a8 arg=7fff1058a0bc 00000080:00200008:0.0:1787826408.370109:0:69475:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff907d84619148), flags 300000 00000080:00200000:0.0:1787826408.373124:0:69479:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826408.373166:0:69479:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826408.373188:0:69479:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 47 lru_type: 1 00010000:00010000:0.0:1787826408.373207:0:69479:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 47 lru_type: 1 00000080:00200000:0.0:1787826408.373216:0:69479:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826408.373225:0:69479:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826408.373230:0:69479:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:0.0:1787826408.373239:0:69479:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142bbfbf8 00010000:00010000:0.0:1787826408.373244:0:69479:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 47 lru_type: 1 00010000:00010000:0.0:1787826408.373254:0:69479:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 48 lru_type: 1 00000080:00200000:0.0:1787826408.373273:0:69479:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826408.373282:0:69479:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=27 00000080:00200000:0.0:1787826408.373288:0:69479:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:0.0:1787826408.373302:0:69479:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), file ffff907d84fb2780, open_flags 18800, mode 0 opened 0 00000080:00200000:0.0:1787826408.373311:0:69479:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=open 00000002:00010000:0.0:1787826408.373331:0:69479:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x208:0x0], intent: open flags 0304001 00010000:00010000:0.0:1787826408.373486:0:69479:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb67 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 69479 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826408.373497:0:69479:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb67 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69479 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:0.0:1787826408.373514:0:69479:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1787826408.373538:0:69479:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90847480 x1874670011476608/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1787826408.373548:0:69479:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb67 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69479 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826408.373585:0:69479:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69479:1874670011476608:192.168.203.104@tcp:101:lfs.0 00000100:00100000:0.0:1787826408.374038:0:69479:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238600 going to sleep for 13 seconds 00000100:00100000:0.0:1787826408.384094:0:69479:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69479:1874670011476608:192.168.203.104@tcp:101:lfs.0 00010000:00010000:0.0:1787826408.384130:0:69479:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb67 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69479 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826408.384148:0:69479:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb67 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 69479 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826408.384156:0:69479:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb67 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 69479 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826408.384165:0:69479:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb67 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 69479 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826408.384175:0:69479:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb67 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 69479 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826408.384181:0:69479:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a400 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:0.0:1787826408.384187:0:69479:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb67 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 69479 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826408.384194:0:69479:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb67 lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 69479 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826408.384202:0:69479:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb67 lrc: 5/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 69479 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826408.384210:0:69479:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826408.384220:0:69479:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb67 lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 69479 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826408.384226:0:69479:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8883a400/0xe5f736940357bb67 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 69479 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:0.0:1787826408.384248:0:69479:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d90847480 x1874670011476608/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826426 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1787826408.384265:0:69479:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x208:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1787826408.384917:0:69479:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:0.0:1787826408.384950:0:69479:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1787826408.384972:0:69479:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 48 lru_type: 1 00010000:00010000:0.0:1787826408.384988:0:69479:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 48 lru_type: 1 00000080:00200000:0.0:1787826408.384996:0:69479:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1787826408.385003:0:69479:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:0.0:1787826408.385007:0:69479:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:0.0:1787826408.385016:0:69479:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142bbfbf8 00010000:00010000:0.0:1787826408.385020:0:69479:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 48 lru_type: 1 00010000:00010000:0.0:1787826408.385030:0:69479:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 49 lru_type: 1 00000080:00200000:0.0:1787826408.385051:0:69479:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:0.0:1787826408.385061:0:69479:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=267 00000080:00200000:0.0:1787826408.385072:0:69479:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 44 00000080:00200008:0.0:1787826408.385083:0:69479:0:(file.c:1090:ll_file_open()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00000080:00200080:0.0:1787826408.385133:0:69479:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8) cmd=c1406916 arg=18a8a20 00000100:00100000:0.0:1787826408.385209:0:69479:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69479:1874670011476736:192.168.203.104@tcp:34:lfs.0 00000100:00100000:0.0:1787826408.385332:0:69479:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d83238600 going to sleep for 13 seconds 00000100:00100000:0.0:1787826408.402574:0:69479:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69479:1874670011476736:192.168.203.104@tcp:34:lfs.0 00000080:00200008:0.0:1787826408.402808:0:69479:0:(file.c:479:ll_file_release()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00010000:00010000:0.0:1787826408.402846:0:69479:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x208:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc107cd expref: -99 pid: 69437 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:2.0:1787826408.458002:0:69481:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:2.0:1787826408.458041:0:69481:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1787826408.458066:0:69481:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 49 lru_type: 1 00010000:00010000:2.0:1787826408.458087:0:69481:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 49 lru_type: 1 00000080:00200000:2.0:1787826408.458098:0:69481:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1787826408.458109:0:69481:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:2.0:1787826408.458113:0:69481:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:2.0:1787826408.458122:0:69481:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142c17bf8 00010000:00010000:2.0:1787826408.458126:0:69481:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 49 lru_type: 1 00010000:00010000:2.0:1787826408.458137:0:69481:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 50 lru_type: 1 00000080:00200000:2.0:1787826408.458158:0:69481:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:2.0:1787826408.458167:0:69481:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=27 00000080:00200000:2.0:1787826408.458173:0:69481:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:2.0:1787826408.458187:0:69481:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), file ffff907d9869d980, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1787826408.458196:0:69481:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=open 00000002:00010000:2.0:1787826408.458215:0:69481:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x208:0x0], intent: open flags 0304001 00010000:00010000:2.0:1787826408.458288:0:69481:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646aa00/0xe5f736940357bb6e lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 69481 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1787826408.458298:0:69481:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646aa00/0xe5f736940357bb6e lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69481 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1787826408.458317:0:69481:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1787826408.458335:0:69481:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d911483c0 x1874670011476992/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/133432 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1787826408.458346:0:69481:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646aa00/0xe5f736940357bb6e lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69481 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826408.458375:0:69481:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69481:1874670011476992:192.168.203.104@tcp:101:lfs.0 00000100:00100000:2.0:1787826408.462093:0:69481:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d87248480 going to sleep for 13 seconds 00000100:00100000:3.0:1787826408.468840:0:69481:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69481:1874670011476992:192.168.203.104@tcp:101:lfs.0 00010000:00010000:3.0:1787826408.468875:0:69481:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646aa00/0xe5f736940357bb6e lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69481 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.468899:0:69481:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646aa00/0xe5f736940357bb6e lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 69481 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.468908:0:69481:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646aa00/0xe5f736940357bb6e lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 69481 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.468916:0:69481:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646aa00/0xe5f736940357bb6e lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x10206400000000 pid: 69481 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.468927:0:69481:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646aa00/0xe5f736940357bb6e lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x206400000000 pid: 69481 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.468934:0:69481:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8646aa00 already unused, calling callback (ffffffffc1aca990) 00010000:00010000:3.0:1787826408.468939:0:69481:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646aa00/0xe5f736940357bb6e lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 69481 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.468948:0:69481:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646aa00/0xe5f736940357bb6e lrc: 4/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x206400000000 pid: 69481 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.468956:0:69481:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646aa00/0xe5f736940357bb6e lrc: 5/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x20e400000000 pid: 69481 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.468965:0:69481:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1787826408.468975:0:69481:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646aa00/0xe5f736940357bb6e lrc: 3/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 69481 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826408.468983:0:69481:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8646aa00/0xe5f736940357bb6e lrc: 0/0,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x1/0x1 rrc: 4 type: IBT flags: 0x4a0f400000000 pid: 69481 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1787826408.469006:0:69481:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0xb, status=-20 req@ffff907d911483c0 x1874670011476992/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2240/608 e 0 to 0 dl 1787826426 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1787826408.469024:0:69481:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000404:0x208:0x0] suppgids 0 -1: rc -20 00000080:00200000:3.0:1787826408.469224:0:69481:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826408.469973:0:69481:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826408.469997:0:69481:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 50 lru_type: 1 00010000:00010000:3.0:1787826408.470014:0:69481:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 50 lru_type: 1 00000080:00200000:3.0:1787826408.470022:0:69481:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826408.470029:0:69481:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826408.470033:0:69481:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:3.0:1787826408.470040:0:69481:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd142c17bf8 00010000:00010000:3.0:1787826408.470044:0:69481:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 50 lru_type: 1 00010000:00010000:3.0:1787826408.470054:0:69481:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 51 lru_type: 1 00000080:00200000:3.0:1787826408.470071:0:69481:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826408.470081:0:69481:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=267 00000080:00200000:3.0:1787826408.470089:0:69481:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 44 00000080:00200008:3.0:1787826408.470099:0:69481:0:(file.c:1090:ll_file_open()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00000080:00200080:3.0:1787826408.470150:0:69481:0:(dir.c:2003:ll_dir_ioctl()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8) cmd=c0086915 arg=1fae3d0 00000100:00100000:3.0:1787826408.471358:0:69481:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69481:1874670011477120:192.168.203.104@tcp:34:lfs.0 00000100:00100000:3.0:1787826408.471600:0:69481:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d98247840 going to sleep for 13 seconds 00000100:00100000:3.0:1787826408.475515:0:69481:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524000 pname:cluuid:pid:xid:nid:opc:job lfs:6f891bcb-15cd-41e0-96e2-db843619c1be:69481:1874670011477120:192.168.203.104@tcp:34:lfs.0 00000080:00200008:3.0:1787826408.475829:0:69481:0:(file.c:479:ll_file_release()) START file d204e.sanity-flr:[0x200000404:0x208:0x0](ffff907d933242c8), flags 304000 00010000:00010000:3.0:1787826408.475872:0:69481:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x208:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc107cd expref: -99 pid: 69437 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:3.0:1787826409.334309:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826410.230241:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011477376:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826410.230388:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011477632:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826410.230430:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1787826410.230437:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011477248:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826410.230466:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011477504:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826410.244722:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011477248:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826410.244749:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011477504:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826410.244763:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011477632:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826410.245670:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011477376:192.168.203.104@tcp:400:kworker.0 00010000:00010000:1.0:1787826410.356793:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826411.386691:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826412.406019:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826413.430476:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1787826414.207916:0:69899:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826414.207955:0:69899:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826414.207979:0:69899:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 51 lru_type: 1 00010000:00010000:1.0:1787826414.207998:0:69899:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 51 lru_type: 1 00000080:00200000:1.0:1787826414.208007:0:69899:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826414.208017:0:69899:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826414.208021:0:69899:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:1.0:1787826414.208030:0:69899:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd141d5bb58 00010000:00010000:1.0:1787826414.208035:0:69899:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 51 lru_type: 1 00010000:00010000:1.0:1787826414.208044:0:69899:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 52 lru_type: 1 00000080:00200000:1.0:1787826414.208064:0:69899:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826414.208072:0:69899:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=24 00000080:00200000:1.0:1787826414.208077:0:69899:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:1.0:1787826414.208089:0:69899:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), flags=8 00000080:00200000:1.0:1787826414.208094:0:69899:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8), intent=getattr 00000002:00010000:1.0:1787826414.208107:0:69899:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000404:0x208:0x0], intent: getattr flags 01 00010000:00010000:1.0:1787826414.208332:0:69899:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 69899 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826414.208341:0:69899:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69899 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826414.208351:0:69899:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 3/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT flags: 0x0 pid: 69899 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1787826414.208397:0:69899:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:69899:1874670011477760:192.168.203.104@tcp:101:rm.0 00000100:00100000:1.0:1787826414.208536:0:69899:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88a3da80 going to sleep for 13 seconds 00000100:00100000:1.0:1787826414.215196:0:69899:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:69899:1874670011477760:192.168.203.104@tcp:101:rm.0 00010000:00010000:1.0:1787826414.215231:0:69899:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 4/1,0 mode: --/CR res: [0x200000404:0x208:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10a35 expref: -99 pid: 69899 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826414.215253:0:69899:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10a35 expref: -99 pid: 69899 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826414.215262:0:69899:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 4/1,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10a35 expref: -99 pid: 69899 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826414.215275:0:69899:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10a35 expref: -99 pid: 69899 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826414.215288:0:69899:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10a35 expref: -99 pid: 69899 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826414.215303:0:69899:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10a35 expref: -99 pid: 69899 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826414.215313:0:69899:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 5/2,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10a35 expref: -99 pid: 69899 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826414.215320:0:69899:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 4/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc10a35 expref: -99 pid: 69899 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1787826414.215333:0:69899:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff907d908a1e00 x1874670011477760/t0(0) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 592/1136 e 0 to 0 dl 1787826432 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1787826414.215345:0:69899:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10a35 expref: -99 pid: 69899 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1787826414.215366:0:69899:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc10a35 expref: -99 pid: 69899 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826414.215389:0:69899:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff907d94b8ec00 type 13 mode 4 res 8589935620/521 (0 0) 00000080:00200000:1.0:1787826414.215397:0:69899:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x208:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1787826414.215464:0:69899:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x209:0x0], updating i_size 2097152 00000080:00200000:1.0:1787826414.215468:0:69899:0:(namei.c:147:ll_iget()) got inode: [0x200000404:0x209:0x0](ffff907d933121c8): rc = 0 00000080:00200000:1.0:1787826414.215487:0:69899:0:(file.c:6854:ll_layout_conf()) [0x200000404:0x209:0x0]: layout version change: 8 -> 8 00000080:00010000:1.0:1787826414.215504:0:69899:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) for lock 0xe5f736940357bb75 00000080:00010000:1.0:1787826414.215519:0:69899:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) 00000080:00010000:1.0:1787826414.215523:0:69899:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357bb75 from it ffff9dd141d5bb20 00010000:00010000:1.0:1787826414.215527:0:69899:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10a35 expref: -99 pid: 69899 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826414.215537:0:69899:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10a35 expref: -99 pid: 69899 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200008:1.0:1787826414.215584:0:69899:0:(file.c:6313:ll_getattr_dentry()) START file f204e.sanity-flr:[0x200000404:0x209:0x0](ffff907d933121c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1787826414.215591:0:69899:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8),name=f204e.sanity-flr 00000002:00010000:1.0:1787826414.215708:0:69899:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000404:0x209:0x0]) in obj [0x200000404:0x208:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826414.215719:0:69899:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10a35 expref: -99 pid: 69899 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826414.215729:0:69899:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 2/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10a35 expref: -99 pid: 69899 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1787826414.215740:0:69899:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000404:0x208:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826414.215745:0:69899:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) 00000080:00010000:1.0:1787826414.215748:0:69899:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000404:0x209:0x0](ffff907d933121c8) for lock 0xe5f736940357bb75 00000080:00010000:1.0:1787826414.215752:0:69899:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357bb75 from it ffff9dd141d5bcb8 00010000:00010000:1.0:1787826414.215754:0:69899:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 3/1,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7918213f4fc10a35 expref: -99 pid: 69899 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826414.215761:0:69899:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7918213f4fc10a35 expref: -99 pid: 69899 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826414.215827:0:69899:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:1.0:1787826414.215836:0:69899:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1787826414.215844:0:69899:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 52 lru_type: 1 00010000:00010000:1.0:1787826414.215853:0:69899:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 52 lru_type: 1 00000080:00200000:1.0:1787826414.215860:0:69899:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1787826414.215864:0:69899:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:1.0:1787826414.215867:0:69899:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:1.0:1787826414.215870:0:69899:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd141d5bc10 00010000:00010000:1.0:1787826414.215873:0:69899:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 52 lru_type: 1 00010000:00010000:1.0:1787826414.215881:0:69899:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 53 lru_type: 1 00000080:00200000:1.0:1787826414.215891:0:69899:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:1.0:1787826414.215898:0:69899:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=24 00000080:00200000:1.0:1787826414.215903:0:69899:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 1 00000080:00200000:1.0:1787826414.215909:0:69899:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204e.sanity-flr, flags=0 00000080:00200000:1.0:1787826414.215913:0:69899:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000404:0x208:0x0](ffff907d933242c8), inode mode 41ed mask 3 00000080:00200000:1.0:1787826414.215919:0:69899:0:(namei.c:2418:ll_unlink()) VFS Op:name=f204e.sanity-flr, dir=[0x200000404:0x208:0x0](ffff907d933242c8) 00010000:00010000:1.0:1787826414.215944:0:69899:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc1091d expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1787826414.215960:0:69899:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x209:0x0]: losing layout lock 00010000:00010000:1.0:1787826414.215973:0:69899:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7918213f4fc10a35 expref: -99 pid: 69899 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1787826414.215981:0:69899:0:(vvp_object.c:117:vvp_conf_set()) [0x200000404:0x209:0x0]: losing layout lock 00010000:00010000:1.0:1787826414.215989:0:69899:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bac6 lrc: 2/0,0 mode: PR/PR res: [0x200000404:0x209:0x0].0x0 bits 0x20/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc1076b expref: -99 pid: 69408 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826414.216034:0:69899:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 1/0,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc1091d expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826414.216042:0:69899:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc10a35 expref: -99 pid: 69899 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826414.216049:0:69899:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bac6 lrc: 1/0,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc1076b expref: -99 pid: 69408 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826414.216055:0:69899:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:1.0:1787826414.216058:0:69899:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0ae00/0xe5f736940357bb44 lrc: 0/0,0 mode: --/CR res: [0x200000404:0x209:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc1091d expref: -99 pid: 69451 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826414.216072:0:69899:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b400/0xe5f736940357bb75 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7918213f4fc10a35 expref: -99 pid: 69899 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1787826414.216080:0:69899:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0b000/0xe5f736940357bac6 lrc: 0/0,0 mode: --/PR res: [0x200000404:0x209:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc1076b expref: -99 pid: 69408 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000020:00100000:1.0:1787826414.216101:0:69899:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1787826414.216110:0:69899:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d908a1e00 x1874670011478016/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 512/2200 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826414.216136:0:69899:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:69899:1874670011478016:192.168.203.104@tcp:36:rm.0 00000100:00100000:1.0:1787826414.216239:0:69899:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d88a3da80 going to sleep for 13 seconds 00000100:00100000:3.0:1787826414.225765:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:1.0:1787826414.225814:0:69899:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job rm:6f891bcb-15cd-41e0-96e2-db843619c1be:69899:1874670011478016:192.168.203.104@tcp:36:rm.0 00000080:00200000:1.0:1787826414.225892:0:69899:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x209:0x0] ignore/verify layout 1/0, layout version 7 restore needed 0 00020000:00200000:1.0:1787826414.225910:0:69899:0:(lov_io.c:693:lov_io_mirror_init()) [0x200000404:0x209:0x0]: flr state: 1, move mirror from 1 to 1, have retried: 0, mirror count: 2 00020000:00200000:1.0:1787826414.225914:0:69899:0:(lov_io.c:750:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1787826414.225922:0:69899:0:(lov_io.c:1253:lov_io_iter_init()) component[2] flags 0x10 00020000:00200000:1.0:1787826414.225936:0:69899:0:(lov_io.c:1253:lov_io_iter_init()) component[3] flags 0x10 00020000:00200000:1.0:1787826414.225944:0:69899:0:(lov_io.c:1253:lov_io_iter_init()) component[4] flags 0x40 00000080:00200000:1.0:1787826414.225972:0:69899:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x209:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1787826414.225982:0:69899:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200000404:0x209:0x0](ffff907d933121c8) 00000100:00100000:2.0:1787826414.226586:0:50445:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:2.0:1787826414.226634:0:50445:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025524992 00000100:00100000:2.0:1787826414.226643:0:50445:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025524992 00000100:00100000:2.0:1787826414.226701:0:50445:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7169 00000100:00100000:2.0:1787826414.226711:0:50445:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d84e76850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:6934:x1874670025524992:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826414.226736:0:50445:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000006df4f474/0xe5f736940357bb36 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x30f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc10901 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826414.226828:0:50445:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1787826414.226836:0:50445:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d84e76850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_002:LOV_OSC_UUID+4:6934:x1874670025524992:12345-192.168.203.104@tcp:104: Request processed in 126us (1089us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826414.226844:0:50445:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7169 00000100:00100000:2.0:1787826414.227537:0:2343:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826414.227567:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025525248 00000100:00100000:2.0:1787826414.227574:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025525248 00000100:00100000:2.0:1787826414.227643:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7170 00000100:00100000:2.0:1787826414.227653:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91148000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:27743:x1874670025525248:12345-192.168.203.104@tcp:104: 00010000:00010000:2.0:1787826414.227667:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x420000030000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1787826414.227719:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:2.0:1787826414.227725:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91148000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:27743:x1874670025525248:12345-192.168.203.104@tcp:104: Request processed in 74us (193us total) trans 0 rc 0/0 00000100:00100000:2.0:1787826414.227731:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7170 00010000:00010000:1.0:1787826414.227855:0:61297:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1787826414.227858:0:47852:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826414.227865:0:47852:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x420000030000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826414.227865:0:61297:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000006df4f474/0xe5f736940357bb36 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x30f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc10901 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826414.227875:0:47852:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83f0bc00 already unused, calling callback (ffffffffc118e410) 00010000:00010000:1.0:1787826414.227875:0:61297:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d863c0000 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826414.227883:0:47852:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x428400030000 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826414.227883:0:61297:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000006df4f474/0xe5f736940357bb36 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x30f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x428400010000 nid: local remote: 0x7918213f4fc10901 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826414.227934:0:47852:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 2/0,0 mode: --/PW res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826414.227934:0:61297:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000006df4f474/0xe5f736940357bb36 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x30f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc10901 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826414.227942:0:47852:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826414.227942:0:61297:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826414.227955:0:47852:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 1/0,0 mode: --/PW res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826414.227963:0:47852:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000095f08f3d/0xe5f736940357bb3d lrc: 0/0,0 mode: --/PW res: [0x240000400:0x329:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->104857599) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7918213f4fc1090f expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826414.227964:0:61297:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000006df4f474/0xe5f736940357bb36 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x30f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc10901 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826414.227972:0:61297:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000006df4f474/0xe5f736940357bb36 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x30f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc10901 expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826414.227987:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826414.227987:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826414.227989:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826414.227989:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826414.227990:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826414.227991:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826414.228006:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011478272:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826414.228070:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011478144:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826414.238594:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011478272:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826414.238635:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011478144:192.168.203.104@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1787826414.455683:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826415.350855:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011478400:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826415.350977:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011478656:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826415.351017:0:2351:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1787826415.351023:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011478528:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826415.351050:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011478784:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826415.356512:0:2351:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff907d986d0000: committing for last_committed 8589943042 gen 1 00000100:00100000:2.0:1787826415.356559:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011478528:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826415.356575:0:2351:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff907d986d0000: committing for last_committed 21474840525 gen 3 00000100:00100000:2.0:1787826415.356591:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011478656:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826415.357802:0:2351:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0001-osc-ffff907d986d0000: committing for last_committed 17179873859 gen 1 00000100:00100000:2.0:1787826415.357830:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011478784:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826415.357843:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011478400:192.168.203.104@tcp:400:kworker.0 00010000:00010000:1.0:1787826415.482552:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1787826415.793255:0:70038:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-flr test 204f: FLR write/stale/resync sel w/forced extension ========================================================== 06:26:54 (1787826414) 00010000:00010000:1.0:1787826416.500788:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826417.525156:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826418.132019:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826418.132197:0:40339:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025526016 00000100:00100000:3.0:1787826418.132210:0:40339:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025526016 00000100:00100000:3.0:1787826418.132433:0:40339:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7171 00000100:00100000:3.0:1787826418.132447:0:40339:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91149e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:36729:x1874670025526016:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826418.132484:0:40339:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000072158d5f/0xe5f736940357bb2f lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x32a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc108fa expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826418.132592:0:40339:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826418.132599:0:40339:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91149e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:36729:x1874670025526016:12345-192.168.203.104@tcp:104: Request processed in 157us (595us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826418.132637:0:40339:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7171 00010000:00010000:0.0:1787826418.132702:0:60483:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826418.132712:0:60483:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000072158d5f/0xe5f736940357bb2f lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x32a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x420000010000 nid: local remote: 0x7918213f4fc108fa expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826418.132722:0:60483:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d863c1600 already unused, calling callback (ffffffffc118e410) 00010000:00010000:0.0:1787826418.132728:0:60483:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000072158d5f/0xe5f736940357bb2f lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x32a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x428400010000 nid: local remote: 0x7918213f4fc108fa expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826418.132788:0:60483:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000072158d5f/0xe5f736940357bb2f lrc: 2/0,0 mode: --/PW res: [0x240000400:0x32a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc108fa expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826418.132796:0:60483:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1787826418.132820:0:60483:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000072158d5f/0xe5f736940357bb2f lrc: 1/0,0 mode: --/PW res: [0x240000400:0x32a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc108fa expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826418.132829:0:60483:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000072158d5f/0xe5f736940357bb2f lrc: 0/0,0 mode: --/PW res: [0x240000400:0x32a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7918213f4fc108fa expref: -99 pid: 69451 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826418.132857:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826418.132859:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826418.132861:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826418.134552:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011478912:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826418.138124:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011478912:192.168.203.104@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1787826418.550654:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826419.572904:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826420.470996:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011479168:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826420.471123:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011479424:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826420.471162:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1787826420.471169:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011479040:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826420.471195:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011479296:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826420.475796:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011479168:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826420.475819:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011479424:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826420.491827:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011479296:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826420.496340:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f411759c-2893-4f83-9e11-e7e6f379c36e:2350:1874670011479040:192.168.203.104@tcp:400:kworker.0 00010000:00010000:0.0:1787826420.599558:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826421.620845:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826422.646574:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826423.673222:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826424.693128:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826425.588853:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011479552:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826425.588875:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011479680:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826425.589155:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011479808:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826425.589335:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011479936:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826425.604723:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011479808:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826425.604775:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011479552:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826425.604852:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011479680:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826425.605149:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011479936:192.168.203.104@tcp:400:kworker.0 00010000:00010000:0.0:1787826425.716888:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826426.746329:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826427.766438:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826428.788944:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826429.817645:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1787826429.839214:0:70422:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826429.839418:0:70422:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826429.839443:0:70422:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 53 lru_type: 1 00010000:00010000:3.0:1787826429.839462:0:70422:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 53 lru_type: 1 00000080:00200000:3.0:1787826429.839470:0:70422:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826429.839480:0:70422:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826429.839484:0:70422:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:3.0:1787826429.839494:0:70422:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd14221fbe0 00010000:00010000:3.0:1787826429.839498:0:70422:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 53 lru_type: 1 00010000:00010000:3.0:1787826429.839513:0:70422:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 54 lru_type: 1 00000080:00200000:3.0:1787826429.839533:0:70422:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 1 00000080:00200000:3.0:1787826429.839549:0:70422:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d204f.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148), flags=1536 00000080:00200000:3.0:1787826429.839552:0:70422:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826429.839557:0:70422:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826429.839564:0:70422:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 54 lru_type: 1 00010000:00010000:3.0:1787826429.839573:0:70422:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 54 lru_type: 1 00000080:00200000:3.0:1787826429.839581:0:70422:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826429.839585:0:70422:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826429.839588:0:70422:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:3.0:1787826429.839592:0:70422:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd14221fce0 00010000:00010000:3.0:1787826429.839594:0:70422:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 54 lru_type: 1 00010000:00010000:3.0:1787826429.839601:0:70422:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 55 lru_type: 1 00000080:00200000:3.0:1787826429.839669:0:70422:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 3 00000080:00200000:3.0:1787826429.839675:0:70422:0:(file.c:6208:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148),name=/ 00000002:00010000:3.0:1787826429.839682:0:70422:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1787826429.839689:0:70422:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 55 lru_type: 1 00010000:00010000:3.0:1787826429.839696:0:70422:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 55 lru_type: 1 00000080:00200000:3.0:1787826429.839703:0:70422:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1787826429.839707:0:70422:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) 00000080:00010000:3.0:1787826429.839710:0:70422:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff907d84619148) for lock 0xe5f736940357ba64 00000080:00010000:3.0:1787826429.839713:0:70422:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xe5f736940357ba64 from it ffff9dd14221fdc8 00010000:00010000:3.0:1787826429.839716:0:70422:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 55 lru_type: 1 00010000:00010000:3.0:1787826429.839724:0:70422:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 56 lru_type: 1 00000080:00200000:3.0:1787826429.839732:0:70422:0:(file.c:6689:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff907d84619148), inode mode 41ed mask 3 00000080:00200000:3.0:1787826429.839738:0:70422:0:(namei.c:2239:do_mkdir()) VFS Op:name=d204f.sanity-flr, dir=[0x200000007:0x1:0x0](ffff907d84619148) 00800000:00000004:3.0:1787826429.839767:0:70422:0:(obd_class.h:1056:obd_statfs_async()) lustre-MDT0000-mdc-ffff907d986d0000: age 1629, max_age 0 00000100:00100000:2.0:1787826429.839877:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83526580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011480064:192.168.203.104@tcp:41:mkdir.0 00010000:00010000:3.0:1787826429.840097:0:70422:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 56 lru_type: 1 00010000:00010000:3.0:1787826429.840152:0:70422:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 56 lru_type: 1 00010000:00010000:3.0:1787826429.840161:0:70422:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1787826429.840164:0:70422:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d8629ce00/0xe5f736940357ba64 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7918213f4fc1061b expref: -99 pid: 69379 timeout: 0 lvb_type: 0 lru_score: 56 lru_type: 1 00000020:00100000:3.0:1787826429.840200:0:70422:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1787826429.840211:0:70422:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d83526940 x1874670011480320/t0(0) o36->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 568/2152 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826429.840232:0:70422:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83526940 pname:cluuid:pid:xid:nid:opc:job mkdir:6f891bcb-15cd-41e0-96e2-db843619c1be:70422:1874670011480320:192.168.203.104@tcp:36:mkdir.0 00000100:00100000:3.0:1787826429.840265:0:70422:0:(client.c:2715:ptlrpc_set_wait()) set ffff907d9999a180 going to sleep for 13 seconds 00000100:00100000:3.0:1787826429.844518:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83526580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011480064:192.168.203.104@tcp:41:mkdir.0 00000100:00100000:0.0:1787826429.844673:0:70422:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83526940 pname:cluuid:pid:xid:nid:opc:job mkdir:6f891bcb-15cd-41e0-96e2-db843619c1be:70422:1874670011480320:192.168.203.104@tcp:36:mkdir.0 00000080:00200000:0.0:1787826429.844725:0:70422:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000404:0x20b:0x0](ffff907d933163c8) 00000080:00200000:0.0:1787826429.844733:0:70422:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000404:0x20b:0x0], updating i_size 4096 00000100:00100000:2.0:1787826430.675519:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:2.0:1787826430.675567:0:2342:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.104@tcp (source: 12345-192.168.203.104@tcp) 00000100:00100000:3.0:1787826430.675643:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025529216 00000100:00100000:3.0:1787826430.675652:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025529216 00000100:00100000:3.0:1787826430.675707:0:40338:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1874670025529344 00000100:00100000:3.0:1787826430.675711:0:40338:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1874670025529344 00000100:00100000:3.0:1787826430.675726:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7172 00000100:00100000:3.0:1787826430.675734:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d91148780 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:f411759c-2893-4f83-9e11-e7e6f379c36e+4:41152:x1874670025529216:12345-192.168.203.104@tcp:104: 00010000:00010000:3.0:1787826430.675751:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.203.104@tcp lock: ffff907d83fc3600/0xe5f736940356837e lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x7918213f4fbdc855 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1787826430.675832:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826430.675838:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d91148780 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:f411759c-2893-4f83-9e11-e7e6f379c36e+4:41152:x1874670025529216:12345-192.168.203.104@tcp:104: Request processed in 105us (330us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826430.675846:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7172 00010000:00010000:1.0:1787826430.675858:0:60482:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:3.0:1787826430.675862:0:40338:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.104@tcp, seq: 7173 00010000:00010000:1.0:1787826430.675865:0:60482:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.104@tcp lock: ffff907d83fc3600/0xe5f736940356837e lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x7918213f4fbdc855 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826430.675868:0:40338:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff907d9114a1c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:f411759c-2893-4f83-9e11-e7e6f379c36e+3:41152:x1874670025529344:12345-192.168.203.104@tcp:104: 00010000:00010000:1.0:1787826430.675873:0:60482:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83fc3600 already unused, calling callback (ffffffffc1319c20) 00010000:00010000:3.0:1787826430.675875:0:40338:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.203.104@tcp lock: ffff907d83fc2800/0xe5f7369403568370 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x7918213f4fbdc847 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:1.0:1787826430.675878:0:60482:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.104@tcp lock: ffff907d83fc3600/0xe5f736940356837e lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1400400000000 nid: local remote: 0x7918213f4fbdc855 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826430.675886:0:60482:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.104@tcp lock: ffff907d83fc3600/0xe5f736940356837e lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1408400000000 nid: local remote: 0x7918213f4fbdc855 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:1.0:1787826430.675893:0:60482:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.104@tcp lock: ffff907d83fc3600/0xe5f736940356837e lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1409400000000 nid: local remote: 0x7918213f4fbdc855 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1787826430.675899:0:60482:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1787826430.675916:0:40338:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1787826430.675922:0:40338:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff907d9114a1c0 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:f411759c-2893-4f83-9e11-e7e6f379c36e+3:41152:x1874670025529344:12345-192.168.203.104@tcp:104: Request processed in 54us (355us total) trans 0 rc 0/0 00000100:00100000:3.0:1787826430.675927:0:40338:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.104@tcp, seq: 7173 00010000:00010000:1.0:1787826430.675931:0:60482:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.203.104@tcp lock: ffff907d83fc3600/0xe5f736940356837e lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x7918213f4fbdc855 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826430.675937:0:60482:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826430.675958:0:60482:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.104@tcp lock: ffff907d83fc3600/0xe5f736940356837e lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x7918213f4fbdc855 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826430.675965:0:60482:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.104@tcp lock: ffff907d83fc3600/0xe5f736940356837e lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x7918213f4fbdc855 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826430.675979:0:60482:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826430.675983:0:60482:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.104@tcp lock: ffff907d83fc2800/0xe5f7369403568370 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x7918213f4fbdc847 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826430.675988:0:60482:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d83fc2800 already unused, calling callback (ffffffffc1319c20) 10000000:00010000:1.0:1787826430.675991:0:60482:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.104@tcp lock: ffff907d83fc2800/0xe5f7369403568370 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x7918213f4fbdc847 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826430.675997:0:60482:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.104@tcp lock: ffff907d83fc2800/0xe5f7369403568370 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x7918213f4fbdc847 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:1.0:1787826430.676003:0:60482:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.104@tcp lock: ffff907d83fc2800/0xe5f7369403568370 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x7918213f4fbdc847 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1787826430.676008:0:60482:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:1.0:1787826430.676022:0:60482:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.203.104@tcp lock: ffff907d83fc2800/0xe5f7369403568370 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x7918213f4fbdc847 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826430.676028:0:60482:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1787826430.676033:0:60482:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.104@tcp lock: ffff907d83fc2800/0xe5f7369403568370 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x7918213f4fbdc847 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826430.676039:0:60482:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.104@tcp lock: ffff907d83fc2800/0xe5f7369403568370 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x7918213f4fbdc847 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1787826430.676052:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826430.676054:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826430.676055:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826430.676069:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011480448:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826430.676130:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1787826430.676137:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011480576:192.168.203.104@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1787826430.678434:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826430.683527:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011480576:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1787826430.683544:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2349:1874670011480448:192.168.203.104@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1787826430.708875:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83526580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011480704:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826430.710311:0:2350:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1787826430.710330:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011480832:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826430.715958:0:2351:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1787826430.715988:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011480960:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826430.716322:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011480832:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826430.720495:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83526580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011480704:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826430.722591:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011480960:192.168.203.104@tcp:400:kworker.0 00010000:00010000:1.0:1787826430.836809:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826431.860993:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:0.0:1787826432.180796:0:40349:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:0.0:1787826432.180804:0:40349:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-client-ffff907d986d0000 from 42 10000000:01000000:0.0:1787826432.180810:0:40349:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:0.0:1787826432.180856:0:40349:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.104@tcp lock: ffff907d8883b600/0xe5f736940357bb7c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826432.180867:0:40349:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.104@tcp lock: ffff907d8883b600/0xe5f736940357bb7c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826432.180881:0:40349:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.104@tcp lock: ffff907d8883b600/0xe5f736940357bb7c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826432.180922:0:40349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:f411759c-2893-4f83-9e11-e7e6f379c36e:40349:1874670011481088:192.168.203.104@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1787826432.181020:0:40349:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6e40 going to sleep for 11 seconds 00010000:00010000:0.0:1787826432.884789:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826433.908819:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826434.932877:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826435.830194:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011481216:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826435.830305:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011481472:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826435.830368:0:2351:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1787826435.830376:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835243c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011481344:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826435.830428:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011481600:192.168.203.104@tcp:400:kworker.0 00010000:00010000:0.0:1787826435.958377:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826436.981237:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826438.004831:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826439.029000:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826440.052737:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826440.948910:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011481856:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826440.948933:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83526580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011481728:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826440.949006:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011482112:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826440.949012:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011481984:192.168.203.104@tcp:400:kworker.0 00010000:00010000:0.0:1787826441.076730:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826442.100806:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826443.124790:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826443.508825:0:40349:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6e40 going to sleep for 1 seconds 00010000:00010000:0.0:1787826444.148767:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826444.532792:0:40349:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6e40 going to sleep for 1 seconds 00010000:00010000:1.0:1787826445.172955:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826445.556730:0:40349:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6e40 going to sleep for 1 seconds 00000100:00100000:2.0:1787826446.068971:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011482368:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826446.069033:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011482240:192.168.203.104@tcp:400:kworker.0 00000100:00100000:2.0:1787826446.069144:0:2350:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011482624:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826446.069354:0:2351:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011482496:192.168.203.104@tcp:400:kworker.0 00010000:00010000:0.0:1787826446.196768:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826446.580715:0:40349:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6e40 going to sleep for 1 seconds 00010000:00010000:1.0:1787826447.220856:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826447.604762:0:40349:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6e40 going to sleep for 1 seconds 00010000:00010000:2.0:1787826448.244782:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:0.0:1787826448.628710:0:40349:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826432/real 1787826432] req@ffff907d90847840 x1874670011481088/t0(0) o101->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 328/344 e 0 to 1 dl 1787826448 ref 2 fl Rpc:XQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787826448.640205:0:40349:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff907d90847840 x1874670011481088/t0(0) o101->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 328/344 e 0 to 1 dl 1787826448 ref 2 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787826448.640221:0:40349:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1787826448.640225:0:40349:0:(import.c:186:ptlrpc_set_import_discon()) MGC192.168.203.104@tcp: Connection to MGS (at 192.168.203.104@tcp) was lost; in progress operations using this service will fail 00000100:00080000:0.0:1787826448.644749:0:40349:0:(import.c:192:ptlrpc_set_import_discon()) import MGS@192.168.203.104@tcp for MGC192.168.203.104@tcp not replayable, auto-deactivating 00000100:00080000:0.0:1787826448.644754:0:40349:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:0.0:1787826448.644763:0:40349:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffff907d90847840 x1874670011481088/t0(0) o101->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 328/344 e 0 to 1 dl 1787826448 ref 2 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787826448.644780:0:40349:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffff907d83524000 x1874670011481216/t0(0) o400->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 224/224 e 0 to 0 dl 1787826451 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787826448.644802:0:40349:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffff907d83526580 x1874670011481728/t0(0) o400->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 224/224 e 0 to 0 dl 1787826456 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787826448.644812:0:40349:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffff907d835252c0 x1874670011482240/t0(0) o400->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 224/224 e 0 to 0 dl 1787826462 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:0.0:1787826448.644822:0:40349:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1787826448.644825:0:40349:0:(pinger.c:193:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:0.0:1787826448.644827:0:40349:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808002 00000100:00080000:0.0:1787826448.644830:0:40349:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00100000:0.0:1787826448.644854:0:40349:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6e40 going to sleep for 0 seconds 00000100:00100000:0.0:1787826448.644864:0:40349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:f411759c-2893-4f83-9e11-e7e6f379c36e:40349:1874670011481088:192.168.203.104@tcp:101:ll_cfg_requeue.0 00010000:00010000:0.0:1787826448.644888:0:40349:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.203.104@tcp lock: ffff907d8883b600/0xe5f736940357bb7c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826448.644899:0:40349:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.203.104@tcp lock: ffff907d8883b600/0xe5f736940357bb7c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:1.0:1787826448.644902:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:0.0:1787826448.644910:0:40349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.104@tcp lock: ffff907d8883b600/0xe5f736940357bb7c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:1.0:1787826448.644910:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826448.644919:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787826448.644919:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011482240:192.168.203.104@tcp:400:kworker.0 00010000:00010000:0.0:1787826448.644924:0:40349:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.203.104@tcp lock: ffff907d8883b600/0xe5f736940357bb7c lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:1.0:1787826448.644939:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1101 00010000:00010000:0.0:1787826448.644949:0:40349:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.104@tcp lock: ffff907d8883b600/0xe5f736940357bb7c lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:1.0:1787826448.644957:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00010000:00010000:0.0:1787826448.644968:0:40349:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883b600 already unused, calling callback (ffffffffc1319c20) 10000000:00010000:0.0:1787826448.644974:0:40349:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.104@tcp lock: ffff907d8883b600/0xe5f736940357bb7c lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826448.644978:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83526580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011481728:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826448.644998:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011482752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1787826448.644999:0:40349:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.104@tcp lock: ffff907d8883b600/0xe5f736940357bb7c lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1787826448.645002:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f411759c-2893-4f83-9e11-e7e6f379c36e:2351:1874670011481216:192.168.203.104@tcp:400:kworker.0 10000000:00010000:0.0:1787826448.645030:0:40349:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.104@tcp lock: ffff907d8883b600/0xe5f736940357bb7c lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1787826448.645038:0:40349:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:0.0:1787826448.645044:0:40349:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826448.645053:0:40349:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.104@tcp lock: ffff907d8883b600/0xe5f736940357bb7c lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826448.645063:0:40349:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.104@tcp lock: ffff907d8883b600/0xe5f736940357bb7c lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1787826448.645074:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011482752:192.168.203.104@tcp:250:kworker.0 10000000:01000000:0.0:1787826448.645093:0:40349:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -5 00000100:00100000:1.0:1787826448.645122:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011482880:192.168.203.104@tcp:400:kworker.0 10000000:01000000:0.0:1787826448.645133:0:40349:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.104@tcp: configuration from log 'lustre-client' failed (-5). 10000000:00020000:0.0:1787826448.645137:0:40349:0:(mgc_request.c:614:do_requeue()) failed processing log: -5 00000100:00100000:1.0:1787826448.645172:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011483136:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826448.645214:0:2349:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1787826448.645223:0:2349:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011483008:192.168.203.104@tcp:400:kworker.0 10000000:01000000:0.0:1787826448.647884:0:40349:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:0.0:1787826448.647890:0:40349:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-sptlrpc-ffff907da0018000 from 1 10000000:01000000:0.0:1787826448.647894:0:40349:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:0.0:1787826448.647929:0:40349:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.104@tcp lock: ffff907d8883a000/0xe5f736940357bb83 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826448.647940:0:40349:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.104@tcp lock: ffff907d8883a000/0xe5f736940357bb83 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826448.647957:0:40349:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.104@tcp lock: ffff907d8883a000/0xe5f736940357bb83 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1787826448.647975:0:40349:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6e40 going to sleep for 0 seconds 00000100:00100000:0.0:1787826448.647983:0:40349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:f411759c-2893-4f83-9e11-e7e6f379c36e:40349:1874670011483264:192.168.203.104@tcp:101:ll_cfg_requeue.0 00010000:00010000:0.0:1787826448.647994:0:40349:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.203.104@tcp lock: ffff907d8883a000/0xe5f736940357bb83 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826448.648002:0:40349:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.203.104@tcp lock: ffff907d8883a000/0xe5f736940357bb83 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826448.648010:0:40349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.104@tcp lock: ffff907d8883a000/0xe5f736940357bb83 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826448.648017:0:40349:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.203.104@tcp lock: ffff907d8883a000/0xe5f736940357bb83 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826448.648025:0:40349:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.104@tcp lock: ffff907d8883a000/0xe5f736940357bb83 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826448.648031:0:40349:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff907d8883a000 already unused, calling callback (ffffffffc1319c20) 10000000:00010000:0.0:1787826448.648035:0:40349:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.104@tcp lock: ffff907d8883a000/0xe5f736940357bb83 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826448.648043:0:40349:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.104@tcp lock: ffff907d8883a000/0xe5f736940357bb83 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:0.0:1787826448.648050:0:40349:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.104@tcp lock: ffff907d8883a000/0xe5f736940357bb83 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1787826448.648056:0:40349:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:0.0:1787826448.648060:0:40349:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1787826448.648065:0:40349:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.104@tcp lock: ffff907d8883a000/0xe5f736940357bb83 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1787826448.648072:0:40349:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.104@tcp lock: ffff907d8883a000/0xe5f736940357bb83 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 40349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1787826448.648087:0:40349:0:(mgc_request.c:1817:mgc_process_log()) Can't get cfg lock: -108 00000100:00080000:0.0:1787826448.648090:0:40349:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:CONNECTING 00010000:00010000:0.0:1787826449.269397:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826450.292848:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1787826451.188820:0:2351:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826435/real 1787826435] req@ffff907d83524f00 x1874670011481600/t0(0) o400->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1787826451 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826451.201429:0:2351:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff907d83524f00 x1874670011481600/t0(0) o400->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1787826451 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826451.201447:0:2351:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1787826451.201451:0:2351:0:(import.c:182:ptlrpc_set_import_discon()) lustre-OST0001-osc-ffff907d986d0000: Connection to lustre-OST0001 (at 192.168.203.104@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:3.0:1787826451.206188:0:2351:0:(import.c:418:ptlrpc_pinger_force()) lustre-OST0001_UUID: waking up pinger s:DISCONN 00000100:00000400:3.0:1787826451.206243:0:2351:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826435/real 1787826435] req@ffff907d83524780 x1874670011481472/t0(0) o400->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1787826451 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826451.206282:0:2351:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff907d83524780 x1874670011481472/t0(0) o400->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1787826451 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826451.206292:0:2351:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1787826451.206297:0:2351:0:(import.c:182:ptlrpc_set_import_discon()) lustre-OST0000-osc-ffff907d986d0000: Connection to lustre-OST0000 (at 192.168.203.104@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:1.0:1787826451.206300:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826451.206302:0:2351:0:(import.c:418:ptlrpc_pinger_force()) lustre-OST0000_UUID: waking up pinger s:DISCONN 00000100:00080000:1.0:1787826451.206321:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1787826451.206349:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011481600:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826451.206361:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83524780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011481472:192.168.203.104@tcp:400:kworker.0 00000100:00080000:1.0:1787826451.206378:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826451.206387:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826451.206390:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826451.206497:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1004 00000100:00080000:1.0:1787826451.206513:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826451.206524:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826451.206543:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011483392:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826451.206547:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011483520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826451.206571:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826451.206577:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826451.206579:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826451.206585:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 952 00000100:00100000:3.0:1787826451.206604:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011483520:192.168.203.104@tcp:8:kworker.0 00000100:00080000:1.0:1787826451.206647:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826451.206654:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826451.206672:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011483648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826451.206694:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826451.206697:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1787826451.206740:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011483648:192.168.203.104@tcp:8:kworker.0 00010000:00010000:0.0:1787826451.316741:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826452.340806:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1787826453.236763:0:2351:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826435/real 1787826435] req@ffff907d835243c0 x1874670011481344/t0(0) o400->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1787826453 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787826453.237352:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826453.237358:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787826453.237415:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826453.237418:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787826453.237420:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826453.237422:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1787826453.237434:0:2348:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1787826453.237447:0:2348:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011483776:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826453.255753:0:2351:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff907d835243c0 x1874670011481344/t0(0) o400->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1787826453 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826453.255773:0:2351:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1787826453.255778:0:2351:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff907d986d0000: Connection to lustre-MDT0000 (at 192.168.203.104@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:3.0:1787826453.255785:0:2351:0:(import.c:418:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00080000:0.0:1787826453.255830:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826453.255837:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787826453.255840:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826453.255845:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826453.255849:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826453.255857:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1109 00000100:00080000:0.0:1787826453.255874:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826453.255885:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835243c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011481344:192.168.203.104@tcp:400:kworker.0 00000008:00100000:0.0:1787826453.255889:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826453.255934:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011483904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826453.255965:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826453.255968:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787826453.255971:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826453.255974:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1787826453.256036:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011483904:192.168.203.104@tcp:38:kworker.0 00010000:00010000:1.0:1787826453.364831:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826454.388794:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826455.412808:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1787826456.308785:0:2351:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826440/real 1787826440] req@ffff907d83525a40 x1874670011481984/t0(0) o400->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1787826456 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:2.0:1787826456.308945:0:2350:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826440/real 1787826440] req@ffff907d83527840 x1874670011482112/t0(0) o400->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1787826456 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787826456.308977:0:2350:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff907d83527840 x1874670011482112/t0(0) o400->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1787826456 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787826456.308985:0:2350:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff907d866bb800 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:2.0:1787826456.309001:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83527840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011482112:192.168.203.104@tcp:400:kworker.0 00000100:00080000:0.0:1787826456.309316:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826456.309324:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787826456.309336:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826456.309339:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787826456.309342:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826456.309345:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787826456.309347:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826456.309350:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1787826456.325527:0:2351:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff907d83525a40 x1874670011481984/t0(0) o400->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1787826456 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826456.325550:0:2351:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff907d98adc800 already not connected (conn 4, was 3): CONNECTING 00000100:00100000:3.0:1787826456.325573:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011481984:192.168.203.104@tcp:400:kworker.0 00010000:00010000:3.0:1787826456.437179:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826457.460802:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826458.357049:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826458.357062:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1787826458.357066:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826458.357070:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1787826458.357074:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826458.357077:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1787826458.357081:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826458.357084:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000400:3.0:1787826458.357097:0:2350:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826440/real 1787826440] req@ffff907d835270c0 x1874670011481856/t0(0) o400->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1787826458 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826458.357138:0:2350:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff907d835270c0 x1874670011481856/t0(0) o400->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1787826458 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826458.357147:0:2350:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff907d8351e800 already not connected (conn 6, was 5): CONNECTING 00000100:00100000:3.0:1787826458.357159:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011481856:192.168.203.104@tcp:400:kworker.0 00000100:00100000:1.0:1787826458.359077:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826448/real 1787826448] req@ffff907d908a2580 x1874670011482752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826458 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826458.359109:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011482752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826458 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826458.359125:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826458.359149:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826458.359165:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011482752:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787826458.359261:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826458.359266:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826458.359269:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826458.359275:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1857 00000100:00080000:2.0:1787826458.359284:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1787826458.359292:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826458.359323:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011484032/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826458.359384:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011484032:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787826458.359495:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826458.359500:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1787826458.359503:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826458.359506:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1787826458.359508:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826458.359511:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:3.0:1787826458.486486:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826459.508785:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826460.532834:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826461.428690:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826451/real 1787826451] req@ffff907d908a21c0 x1874670011483648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826461 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826461.428724:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011483648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826461 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826461.428734:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826451/real 1787826451] req@ffff907d908a2940 x1874670011483520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826461 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826461.428747:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011483520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826461 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826461.428770:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826461.428797:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826461.428802:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011483648:192.168.203.104@tcp:8:kworker.0 00000100:00080000:1.0:1787826461.428808:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826461.428813:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826461.428816:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011483520:192.168.203.104@tcp:8:kworker.0 00000100:00080000:0.0:1787826461.428895:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826461.428901:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787826461.428904:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826461.428907:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787826461.428909:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826461.428912:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826461.428915:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826461.428920:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1860 00000100:00080000:0.0:1787826461.428930:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 5s 00000100:00080000:0.0:1787826461.428937:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826461.428947:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826461.428976:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011484160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826461.428998:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826461.429000:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826461.429009:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826461.429013:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1860 00000100:00080000:0.0:1787826461.429017:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 5s 00000100:00080000:0.0:1787826461.429020:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826461.429025:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826461.429034:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011484288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826461.429098:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011484160:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826461.429199:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011484288:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826461.557304:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1787826462.452733:0:2351:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826446/real 1787826446] req@ffff907d83525680 x1874670011482496/t0(0) o400->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1787826462 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:2.0:1787826462.453373:0:2350:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826446/real 1787826446] req@ffff907da562cb40 x1874670011482624/t0(0) o400->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1787826462 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787826462.453598:0:2350:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff907da562cb40 x1874670011482624/t0(0) o400->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1787826462 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787826462.453632:0:2350:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff907d866bb800 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:2.0:1787826462.453656:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011482624:192.168.203.104@tcp:400:kworker.0 00000100:00100000:3.0:1787826462.472363:0:2351:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff907d83525680 x1874670011482496/t0(0) o400->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1787826462 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826462.472377:0:2351:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff907d98adc800 already not connected (conn 5, was 3): CONNECTING 00000100:00100000:3.0:1787826462.472393:0:2351:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d83525680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2351:1874670011482496:192.168.203.104@tcp:400:kworker.0 00010000:00010000:0.0:1787826462.580735:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826463.476737:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826463.476748:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787826463.476751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826463.476764:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787826463.476767:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826463.476770:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1787826463.476773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826463.476776:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1787826463.476876:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826453/real 1787826453] req@ffff907d908470c0 x1874670011483904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826463 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826463.476905:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011483904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826463 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826463.476919:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826463.476935:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826463.476939:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011483904:192.168.203.104@tcp:38:kworker.0 00000100:00080000:3.0:1787826463.477025:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826463.477030:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787826463.477033:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826463.477036:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787826463.477039:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826463.477046:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1862 00000100:00080000:3.0:1787826463.477056:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1787826463.477063:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826463.477078:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826463.477107:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011484416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826463.477128:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826463.477132:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787826463.477134:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826463.477137:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1787826463.477175:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011484416:192.168.203.104@tcp:38:kworker.0 00010000:00010000:1.0:1787826463.604888:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1787826464.500734:0:2350:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826446/real 1787826446] req@ffff907d835261c0 x1874670011482368/t0(0) o400->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1787826464 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787826464.500770:0:2350:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff907d835261c0 x1874670011482368/t0(0) o400->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1787826464 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787826464.500780:0:2350:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff907d8351e800 already not connected (conn 7, was 5): CONNECTING 00000100:00100000:2.0:1787826464.500791:0:2350:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d835261c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2350:1874670011482368:192.168.203.104@tcp:400:kworker.0 00010000:00010000:3.0:1787826464.629055:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:0.0:1787826465.012789:0:2349:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826448/real 1787826448] req@ffff907d908a1680 x1874670011483008/t0(0) o400->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1787826464 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787826465.012813:0:2349:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff907d908a1680 x1874670011483008/t0(0) o400->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1787826464 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787826465.012818:0:2349:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff907d98adc800 already not connected (conn 5, was 3): CONNECTING 00000100:00000400:0.0:1787826465.012823:0:2349:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826448/real 1787826448] req@ffff907d908a0f00 x1874670011483136/t0(0) o400->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1787826464 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787826465.012834:0:2349:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff907d908a0f00 x1874670011483136/t0(0) o400->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 224/224 e 0 to 1 dl 1787826464 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787826465.012840:0:2349:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff907d866bb800 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:0.0:1787826465.012856:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011483008:192.168.203.104@tcp:400:kworker.0 00000100:00100000:0.0:1787826465.012865:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011483136:192.168.203.104@tcp:400:kworker.0 00010000:00010000:0.0:1787826465.653042:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:2.0:1787826465.908781:0:2339:0:(socklnd_cb.c:2254:ksocknal_find_timed_out_conn()) A connection with 12345-192.168.203.104@tcp (192.168.203.104:988) timed out; the network or node may be down. 00000100:00000400:0.0:1787826466.036763:0:2349:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826448/real 1787826448] req@ffff907d908a30c0 x1874670011482880/t0(0) o400->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1787826466 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787826466.036797:0:2349:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff907d908a30c0 x1874670011482880/t0(0) o400->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1787826466 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787826466.036807:0:2349:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff907d8351e800 already not connected (conn 7, was 5): CONNECTING 00000100:00100000:0.0:1787826466.036823:0:2349:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:6f891bcb-15cd-41e0-96e2-db843619c1be:2349:1874670011482880:192.168.203.104@tcp:400:kworker.0 00000100:00080000:2.0:1787826466.486871:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826466.486882:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1787826466.486885:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826466.486888:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1787826466.486891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826466.486894:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1787826466.486897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826466.486900:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:3.0:1787826466.676805:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826467.700912:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826468.533530:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826468.533541:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1787826468.533544:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826468.533547:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1787826468.533550:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826468.533677:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1787826468.533680:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826468.533682:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:2.0:1787826468.724893:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:0.0:1787826469.109096:0:40349:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.104@tcp: configuration from log 'lustre-sptlrpc' failed (-108). 10000000:00020000:0.0:1787826469.109106:0:40349:0:(mgc_request.c:614:do_requeue()) failed processing log: -108 00000100:00000400:0.0:1787826469.620855:0:2348:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826451/real 1787826451] req@ffff907d908a12c0 x1874670011483392/t0(0) o400->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1787826469 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787826469.620887:0:2348:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff907d908a12c0 x1874670011483392/t0(0) o400->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1787826469 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787826469.620896:0:2348:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff907d8351e800 already not connected (conn 7, was 5): CONNECTING 00000100:00100000:0.0:1787826469.620909:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011483392:192.168.203.104@tcp:400:kworker.0 00010000:00010000:1.0:1787826469.748816:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826470.772759:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:0.0:1787826471.668724:0:2348:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826453/real 1787826453] req@ffff907d90847840 x1874670011483776/t0(0) o400->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1787826471 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787826471.685991:0:2348:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff907d90847840 x1874670011483776/t0(0) o400->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 224/224 e 0 to 1 dl 1787826471 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787826471.686010:0:2348:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff907d8351e800 already not connected (conn 7, was 5): CONNECTING 00000100:00100000:0.0:1787826471.686028:0:2348:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:6f891bcb-15cd-41e0-96e2-db843619c1be:2348:1874670011483776:192.168.203.104@tcp:400:kworker.0 00010000:00010000:1.0:1787826471.796837:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826472.820761:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826473.717194:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826473.717206:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1787826473.717208:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826458/real 1787826458] req@ffff907d91148b40 x1874670011484032/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826473 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787826473.717210:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826473.717218:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1787826473.717225:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826473.717240:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1787826473.717247:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011484032/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826473 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787826473.717247:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826473.717256:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1787826473.717275:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826473.717288:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826473.717293:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011484032:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787826473.717294:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826473.717301:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826473.717304:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826473.717312:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1867 00000100:00080000:2.0:1787826473.717322:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1787826473.717329:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826473.717365:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011484544/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826473.717392:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826473.717396:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1787826473.717398:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826473.717400:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1787826473.717403:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826473.717406:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1787826473.717422:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011484544:192.168.203.104@tcp:250:kworker.0 00010000:00010000:3.0:1787826473.844763:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826474.868957:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826475.892838:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826476.788945:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826461/real 1787826461] req@ffff907d90844b40 x1874670011484160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826476 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826476.788980:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011484160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826476 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826476.788990:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826461/real 1787826461] req@ffff907d90847c00 x1874670011484288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826476 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826476.789000:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011484288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826476 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826476.789018:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826476.789039:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826476.789044:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011484160:192.168.203.104@tcp:8:kworker.0 00000100:00080000:1.0:1787826476.789050:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826476.789073:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826476.789077:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011484288:192.168.203.104@tcp:8:kworker.0 00000100:00080000:3.0:1787826476.789093:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826476.789099:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787826476.789102:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826476.789105:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787826476.789107:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826476.789111:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787826476.789114:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826476.789121:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1870 00000100:00080000:3.0:1787826476.789270:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1787826476.789279:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826476.789289:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826476.789315:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011484672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826476.789338:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826476.789342:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787826476.789343:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826476.789347:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1870 00000100:00080000:3.0:1787826476.789351:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1787826476.789355:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826476.789359:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826476.789421:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011484800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826476.789456:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011484672:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826476.789549:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011484800:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787826476.916904:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826477.940807:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787826478.836730:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787826478.836736:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787826463/real 1787826463] req@ffff907da562de00 x1874670011484416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826478 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826478.836739:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787826478.836744:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826478.836751:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787826478.836757:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826478.836761:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787826478.836764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826478.836767:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1787826478.836784:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011484416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826478 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826478.836807:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826478.836833:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826478.836838:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011484416:192.168.203.104@tcp:38:kworker.0 00000100:00080000:3.0:1787826478.836870:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826478.836875:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787826478.836878:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826478.836892:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787826478.836895:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826478.836901:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1872 00000100:00080000:3.0:1787826478.836912:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1787826478.836920:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826478.836931:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826478.837019:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011484928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826478.837049:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826478.837053:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787826478.837056:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826478.837059:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1787826478.837084:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011484928:192.168.203.104@tcp:38:kworker.0 00010000:00010000:0.0:1787826478.964762:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826479.988935:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826481.012767:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787826481.845038:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826481.845052:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787826481.845056:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826481.845059:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787826481.845063:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826481.845067:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787826481.845071:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826481.845074:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:3.0:1787826482.036848:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826483.060783:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826483.894351:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826483.894362:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1787826483.894365:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826483.894369:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1787826483.894371:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826483.894374:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1787826483.894377:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826483.894380:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:3.0:1787826484.084779:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826485.109213:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826486.135056:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826487.156767:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826488.180874:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826489.077184:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826489.077192:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1787826489.077195:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826489.077198:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1787826489.077201:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826489.077204:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1787826489.077207:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826489.077211:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:2.0:1787826489.204864:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787826490.101024:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.104:988 00000400:00000100:0.0:1787826490.101071:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787826490.101087:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826490.101118:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826490.101129:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826490.101140:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:3.0:1787826490.101143:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826476/real 1787826490] req@ffff907da562e1c0 x1874670011484672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826490.101169:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011484672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826490.101180:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826490.101204:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826490.101207:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011484672:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826490.101214:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826476/real 1787826490] req@ffff907da562da40 x1874670011484800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826490.101223:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011484800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826490.101228:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826490.101231:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826490.101234:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011484800:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826490.101237:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826473/real 1787826490] req@ffff907d911492c0 x1874670011484544/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826490.101249:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011484544/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826490.101256:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826490.101259:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826490.101261:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011484544:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826490.101278:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826478/real 1787826490] req@ffff907da562f840 x1874670011484928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826490.101285:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011484928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826490.101290:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826490.101292:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826490.101294:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011484928:192.168.203.104@tcp:38:kworker.0 00000100:00080000:0.0:1787826490.101735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826490.101741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826490.101743:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826490.101749:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1882 00000100:00080000:0.0:1787826490.101756:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 15s 00000100:00080000:0.0:1787826490.101762:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826490.101790:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847840 x1874670011485056/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826490.101811:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826490.101815:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826490.101817:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826490.101820:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1887 00000100:00080000:0.0:1787826490.101825:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 15s 00000100:00080000:0.0:1787826490.101829:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826490.101836:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011485056:192.168.203.104@tcp:250:kworker.0 00000008:00100000:0.0:1787826490.101839:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826490.101855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908461c0 x1874670011485184/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826490.101866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826490.101869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826490.101871:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826490.101875:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1885 00000100:00080000:0.0:1787826490.101878:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 15s 00000100:00080000:0.0:1787826490.101883:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826490.101888:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787826490.101894:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787826490.101899:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011485312/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826490.101910:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826490.101913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826490.101915:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826490.101919:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1885 00000100:00080000:0.0:1787826490.101924:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 15s 00000100:00100000:3.0:1787826490.101924:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826490/real 1787826490] req@ffff907d90847840 x1874670011485056/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787826490.101932:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826490.101942:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826490.101958:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011485440/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826490.101958:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847840 x1874670011485056/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826490.101993:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826490.101999:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826490.102003:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011485056:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826490.102018:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011485184:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826490.102041:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826490.102053:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826490/real 1787826490] req@ffff907d908461c0 x1874670011485184/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826490.102065:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908461c0 x1874670011485184/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826490.102071:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826490.102074:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826490.102077:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011485184:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826490.102082:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011485312:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826490.102102:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826490.102112:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826490/real 1787826490] req@ffff907d90846580 x1874670011485312/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826490.102122:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011485312/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826490.102128:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826490.102131:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826490.102134:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011485312:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826490.102140:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011485440:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826490.102155:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826490.102164:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826490/real 1787826490] req@ffff907d90847c00 x1874670011485440/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826490.102224:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011485440/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826490.102233:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826490.102237:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826490.102240:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011485440:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787826490.228775:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826491.253814:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826492.277980:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826493.301062:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826494.324858:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826495.220742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826495.220752:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826495.220756:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826495.220765:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1898 00000100:00080000:2.0:1787826495.220788:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826495.220796:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826495.220832:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011485568/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826495.220864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826495.220869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826495.220870:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826495.220874:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1898 00000100:00080000:2.0:1787826495.220878:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826495.220881:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826495.220890:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826495.220902:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011485696/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826495.220917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826495.220919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826495.220921:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826495.220925:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1898 00000100:00080000:2.0:1787826495.220940:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826495.220944:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826495.220948:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826495.220958:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011485824/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826495.220967:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826495.220970:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826495.220971:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826495.220974:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1898 00000100:00080000:2.0:1787826495.220977:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826495.220981:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826495.220985:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826495.220993:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011485952/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826495.221119:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011485568:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826495.221207:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011485696:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826495.221237:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011485824:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826495.221261:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011485952:192.168.203.104@tcp:8:kworker.0 00010000:00010000:0.0:1787826495.348773:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826496.372836:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787826496.757196:0:2337:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.104:988 00000400:00000100:0.0:1787826496.757254:0:2337:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787826496.757275:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826496.757319:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826496.757328:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826496.757337:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:1.0:1787826496.757412:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826495/real 1787826496] req@ffff907d9114a940 x1874670011485568/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826496.757445:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011485568/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826496.757457:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826496.757462:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826496.757466:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011485568:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826496.757474:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826495/real 1787826496] req@ffff907d91148f00 x1874670011485696/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826496.757487:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011485696/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826496.757496:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826496.757499:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826496.757502:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011485696:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826496.757509:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826495/real 1787826496] req@ffff907d9114ad00 x1874670011485824/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826496.757521:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011485824/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826496.757528:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826496.757532:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826496.757535:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011485824:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826496.757541:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826495/real 1787826496] req@ffff907d91149a40 x1874670011485952/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826496.757552:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011485952/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826496.757559:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826496.757563:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826496.757566:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011485952:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787826497.396928:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826498.420764:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826499.444762:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826500.341028:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826500.341042:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826500.341057:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826500.341069:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1904 00000100:00080000:0.0:1787826500.341082:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826500.341089:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826500.341125:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011486080/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826500.341162:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826500.341166:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826500.341168:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826500.341172:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1904 00000100:00080000:0.0:1787826500.341176:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826500.341179:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826500.341189:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826500.341203:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011486208/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826500.341213:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787826500.341218:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011486080:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787826500.341219:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826500.341223:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826500.341230:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1904 00000100:00080000:0.0:1787826500.341234:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826500.341237:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826500.341242:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826500.341253:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011486336/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826500.341262:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826500.341266:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826500.341268:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826500.341271:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1904 00000100:00080000:0.0:1787826500.341275:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826500.341279:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826500.341284:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826500.341295:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011486464/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826500.341340:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011486208:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826500.341369:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011486336:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826500.341390:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011486464:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787826500.468994:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826501.492864:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826502.516829:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787826503.413221:0:2338:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.104:988 00000400:00000100:0.0:1787826503.413269:0:2338:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787826503.413285:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826503.413327:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826503.413337:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826503.413345:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:1.0:1787826503.413681:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826500/real 1787826503] req@ffff907d908470c0 x1874670011486208/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826503.413708:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011486208/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826503.413723:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826503.413730:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826503.413736:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011486208:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826503.413747:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826500/real 1787826503] req@ffff907d90844000 x1874670011486336/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826503.413766:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011486336/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826503.413774:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826503.413778:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826503.413781:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011486336:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826503.413787:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826500/real 1787826503] req@ffff907d908443c0 x1874670011486464/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826503.413798:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011486464/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826503.413805:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826503.413808:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826503.413811:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011486464:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826503.413817:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826500/real 1787826503] req@ffff907d90844b40 x1874670011486080/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826503.413830:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011486080/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826503.413837:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826503.413840:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826503.413843:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011486080:192.168.203.104@tcp:250:kworker.0 00010000:00010000:0.0:1787826503.540761:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826504.564835:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826505.460852:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826505.460872:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826505.460876:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826505.460887:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1909 00000100:00080000:1.0:1787826505.460898:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826505.460906:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826505.460942:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011486592/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826505.460974:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826505.460978:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826505.460980:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826505.460984:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1909 00000100:00080000:1.0:1787826505.460987:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826505.460991:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826505.461000:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826505.461012:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011486720/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826505.461028:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826505.461031:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826505.461032:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826505.461036:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1909 00000100:00080000:1.0:1787826505.461039:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826505.461042:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826505.461047:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826505.461056:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011486848/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826505.461065:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826505.461068:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826505.461070:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826505.461084:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1909 00000100:00080000:1.0:1787826505.461088:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826505.461091:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826505.461096:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826505.461105:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011486976/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826505.461199:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011486592:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787826505.461270:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826505.461323:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826505/real 1787826505] req@ffff907d908a2940 x1874670011486592/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826505.461339:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011486592/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826505.461349:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826505.461352:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826505.461356:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011486592:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826505.461364:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011486720:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826505.461385:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826505.461399:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826505/real 1787826505] req@ffff907d908a21c0 x1874670011486720/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826505.461409:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011486720/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826505.461417:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826505.461420:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826505.461423:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011486720:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826505.461428:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011486848:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826505.461447:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826505.461457:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826505/real 1787826505] req@ffff907d908a2580 x1874670011486848/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826505.461467:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011486848/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826505.461474:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826505.461476:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826505.461479:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011486848:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826505.461484:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011486976:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826505.461499:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826505.461509:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826505/real 1787826505] req@ffff907d908a2d00 x1874670011486976/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826505.461518:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011486976/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826505.461526:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826505.461528:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826505.461530:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011486976:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787826505.589044:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826506.612771:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826507.636767:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826508.660759:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826509.684789:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826510.580715:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826510.580723:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826510.580727:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826510.580735:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1914 00000100:00080000:1.0:1787826510.580744:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826510.580751:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826510.580780:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011487104/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826510.580803:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826510.580805:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826510.580807:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826510.580809:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1914 00000100:00080000:1.0:1787826510.580812:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826510.580817:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826510.580842:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826510.580853:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011487232/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826510.580861:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826510.580863:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826510.580864:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826510.580867:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1914 00000100:00080000:1.0:1787826510.580870:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826510.580872:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826510.580877:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826510.580888:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011487104:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826510.580891:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011487360/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826510.580903:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826510.580907:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826510.580909:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826510.580915:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1914 00000100:00080000:1.0:1787826510.580920:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826510.580926:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826510.580931:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826510.580948:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011487488/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826510.580983:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011487232:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826510.581087:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011487360:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826510.581113:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011487488:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826510.708791:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826511.732807:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826512.756762:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787826513.653069:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.104:988 00000400:00000100:0.0:1787826513.653121:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787826513.653137:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826513.653180:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826513.653189:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826513.653196:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:3.0:1787826513.653413:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826510/real 1787826513] req@ffff907d908a0000 x1874670011487360/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826513.653435:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011487360/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826513.653445:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826513.653450:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826513.653453:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011487360:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826513.653461:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826510/real 1787826513] req@ffff907d908a2d00 x1874670011487488/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826513.653472:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011487488/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826513.653479:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826513.653482:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826513.653485:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011487488:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826513.653491:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826510/real 1787826513] req@ffff907d908a1e00 x1874670011487104/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826513.653502:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011487104/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826513.653509:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826513.653512:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826513.653515:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011487104:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826513.653521:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826510/real 1787826513] req@ffff907d908a3840 x1874670011487232/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826513.653531:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011487232/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826513.653538:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826513.653541:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826513.653544:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011487232:192.168.203.104@tcp:38:kworker.0 00010000:00010000:2.0:1787826513.780902:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826514.804764:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826515.700747:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826515.700761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826515.700765:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826515.700781:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1919 00000100:00080000:2.0:1787826515.700802:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826515.700810:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826515.700844:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011487616/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826515.700880:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826515.700884:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826515.700885:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826515.700890:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1919 00000100:00080000:2.0:1787826515.700894:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826515.700898:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826515.700907:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826515.700931:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011487744/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826515.700940:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826515.700943:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826515.700945:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826515.700948:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1919 00000100:00080000:2.0:1787826515.700952:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826515.700955:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826515.700960:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826515.700970:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011487872/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826515.700978:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826515.700981:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826515.700983:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826515.700986:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1919 00000100:00080000:2.0:1787826515.700989:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826515.700992:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826515.700996:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826515.701004:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011488000/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826515.701092:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011487616:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787826515.701163:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826515.701265:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826515/real 1787826515] req@ffff907d9114a580 x1874670011487616/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826515.701282:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011487616/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826515.701292:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826515.701295:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826515.701299:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011487616:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826515.701306:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011487744:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826515.701329:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826515.701341:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826515/real 1787826515] req@ffff907d9114b840 x1874670011487744/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826515.701352:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011487744/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826515.701359:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826515.701362:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826515.701365:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011487744:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826515.701370:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011487872:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826515.701390:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826515.701400:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826515/real 1787826515] req@ffff907d91149680 x1874670011487872/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826515.701411:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011487872/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826515.701424:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826515.701429:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826515.701431:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011487872:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826515.701436:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011488000:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826515.701459:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826515.701524:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826515/real 1787826515] req@ffff907d9114bc00 x1874670011488000/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826515.701534:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011488000/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826515.701541:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826515.701544:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826515.701547:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011488000:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826515.828756:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826516.852759:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826517.876804:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826518.901998:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826519.924784:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826520.820714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826520.820725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826520.820729:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826520.820736:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1924 00000100:00080000:2.0:1787826520.820745:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826520.820761:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826520.820794:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011488128/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826520.820877:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011488128:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787826520.820939:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787826520.820944:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826520.820955:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826520.820960:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826520.820970:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1924 00000100:00100000:3.0:1787826520.820977:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826520/real 1787826520] req@ffff907d911483c0 x1874670011488128/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787826520.820984:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787826520.820991:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011488128/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787826520.820997:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00080000:3.0:1787826520.821008:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826520.821012:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826520.821015:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011488128:192.168.203.104@tcp:250:kworker.0 00000008:00100000:2.0:1787826520.821018:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826520.821055:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011488256/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826520.821082:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826520.821085:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826520.821086:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826520.821090:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1924 00000100:00080000:2.0:1787826520.821094:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826520.821099:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826520.821105:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826520.821115:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011488256:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787826520.821124:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011488384/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826520.821139:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826520.821141:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826520.821143:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826520.821146:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1924 00000100:00080000:2.0:1787826520.821150:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787826520.821155:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787826520.821156:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826520.821164:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826520.821176:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011488512/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826520.821183:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826520/real 1787826520] req@ffff907d9114b480 x1874670011488256/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826520.821231:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011488256/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826520.821241:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826520.821246:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826520.821250:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011488256:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826520.821268:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011488384:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826520.821288:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826520.821296:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826520/real 1787826520] req@ffff907d911483c0 x1874670011488384/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826520.821302:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011488384/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826520.821307:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826520.821309:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826520.821311:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011488384:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826520.821315:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011488512:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826520.821324:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826520.821341:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826520/real 1787826520] req@ffff907d9114bc00 x1874670011488512/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826520.821352:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011488512/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826520.821358:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826520.821361:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826520.821365:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011488512:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826520.948843:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826521.972801:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826522.996718:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826524.020942:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826525.044772:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826525.940775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826525.940782:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826525.940785:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826525.940793:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1929 00000100:00080000:1.0:1787826525.940803:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826525.940809:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826525.940839:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011488640/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826525.940868:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826525.940871:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826525.940873:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826525.940878:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1929 00000100:00080000:1.0:1787826525.940881:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826525.940885:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826525.940894:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826525.940907:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011488768/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826525.940917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826525.940920:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826525.940922:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826525.940925:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1929 00000100:00080000:1.0:1787826525.940928:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826525.940930:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826525.940933:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826525.940940:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011488896/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826525.940945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826525.940947:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826525.940948:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826525.940951:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1929 00000100:00080000:1.0:1787826525.940954:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826525.940956:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826525.940960:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826525.940969:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011489024/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826525.941066:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011488640:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826525.941159:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011488768:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826525.941195:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011488896:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826525.941221:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011489024:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787826526.069842:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826527.092875:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826528.116757:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787826529.013095:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.104:988 00000400:00000100:2.0:1787826529.013158:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787826529.013182:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787826529.013251:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787826529.013266:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787826529.013283:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:3.0:1787826529.013284:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826525/real 1787826529] req@ffff907d908a2580 x1874670011488640/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826529.013375:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011488640/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826529.013387:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826529.013392:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826529.013396:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011488640:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826529.013403:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826525/real 1787826529] req@ffff907d908a21c0 x1874670011488768/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826529.013415:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011488768/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826529.013422:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826529.013424:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826529.013427:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011488768:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826529.013433:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826525/real 1787826529] req@ffff907d908a2940 x1874670011488896/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826529.013443:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011488896/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826529.013449:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826529.013452:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826529.013455:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011488896:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826529.013461:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826525/real 1787826529] req@ffff907d908a12c0 x1874670011489024/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826529.013472:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011489024/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826529.013478:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826529.013484:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826529.013488:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011489024:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826529.140771:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826530.165660:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826531.060755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826531.060764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826531.060769:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826531.060778:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1934 00000100:00080000:2.0:1787826531.060789:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826531.060796:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826531.060829:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011489152/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826531.060856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826531.060859:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826531.060860:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826531.060865:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1934 00000100:00080000:2.0:1787826531.060868:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826531.060872:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826531.060882:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826531.060901:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011489152:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787826531.060902:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011489280/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826531.060917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826531.060920:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826531.060922:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826531.060926:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1934 00000100:00080000:2.0:1787826531.060929:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826531.060932:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826531.060937:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826531.060948:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011489408/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826531.060956:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826531.060959:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826531.060961:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826531.060964:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1934 00000100:00080000:2.0:1787826531.060967:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787826531.060972:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787826531.060973:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826531.060980:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826531.060992:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011489536/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826531.061017:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826531/real 1787826531] req@ffff907d91149680 x1874670011489152/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826531.061043:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011489152/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826531.061064:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826531.061068:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826531.061072:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011489152:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826531.061093:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011489280:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826531.061126:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826531.061139:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826531/real 1787826531] req@ffff907d9114b840 x1874670011489280/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826531.061152:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011489280/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826531.061159:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826531.061162:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826531.061165:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011489280:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826531.061171:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011489408:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826531.061208:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826531.061221:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826531/real 1787826531] req@ffff907d9114a580 x1874670011489408/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826531.061237:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011489408/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826531.061244:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826531.061247:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826531.061255:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011489408:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826531.061260:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011489536:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826531.061278:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826531.061296:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826531/real 1787826531] req@ffff907d91149a40 x1874670011489536/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826531.061307:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011489536/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826531.061314:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826531.061317:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826531.061320:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011489536:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787826531.188815:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826532.212781:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826533.236780:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826534.260802:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826535.284876:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826536.180791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826536.180806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826536.180811:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826536.180825:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1939 00000100:00080000:0.0:1787826536.180840:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826536.180848:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826536.180897:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011489664/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826536.180929:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826536.180934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826536.180935:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826536.180939:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1939 00000100:00080000:0.0:1787826536.180943:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826536.180947:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826536.180968:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826536.180981:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011489792/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826536.180998:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826536.181001:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826536.181003:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826536.181007:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1939 00000100:00080000:0.0:1787826536.181011:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826536.181015:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826536.181021:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826536.181028:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011489664:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787826536.181039:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011489920/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826536.181052:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826536.181056:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826536.181058:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826536.181063:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1939 00000100:00080000:0.0:1787826536.181066:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826536.181070:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826536.181076:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826536.181088:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011490048/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787826536.181115:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826536.181163:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826536/real 1787826536] req@ffff907d90845e00 x1874670011489664/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826536.181180:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011489664/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826536.181193:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826536.181197:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826536.181201:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011489664:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826536.181209:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011489792:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826536.181233:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826536.181246:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826536/real 1787826536] req@ffff907d90844780 x1874670011489792/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826536.181255:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011489792/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826536.181262:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826536.181265:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826536.181269:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011489792:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826536.181291:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011489920:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826536.181313:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826536.181325:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826536/real 1787826536] req@ffff907d90845680 x1874670011489920/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826536.181335:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011489920/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826536.181342:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826536.181346:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826536.181348:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011489920:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826536.181355:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011490048:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826536.181384:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826536.181395:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826536/real 1787826536] req@ffff907d90846d00 x1874670011490048/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826536.181405:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011490048/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826536.181413:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826536.181416:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826536.181419:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011490048:192.168.203.104@tcp:8:kworker.0 00010000:00010000:0.0:1787826536.309785:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826537.332777:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826538.356792:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826539.380760:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826540.404888:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826541.300786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826541.300801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826541.300806:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826541.300818:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1945 00000100:00080000:1.0:1787826541.300838:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826541.300847:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826541.300897:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a30c0 x1874670011490176/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826541.300924:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826541.300927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826541.300929:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826541.300933:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1945 00000100:00080000:1.0:1787826541.300937:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826541.300941:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826541.300954:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826541.300969:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011490304/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826541.300987:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011490176:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787826541.300988:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826541.300997:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826541.301004:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826541.301011:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1945 00000100:00080000:1.0:1787826541.301015:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826541.301019:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826541.301024:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826541.301040:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011490432/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826541.301051:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826541.301054:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826541.301057:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826541.301060:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1945 00000100:00080000:1.0:1787826541.301063:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826541.301066:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826541.301070:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826541.301083:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011490560/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787826541.301086:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826541.301153:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826541/real 1787826541] req@ffff907d908a30c0 x1874670011490176/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826541.301174:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a30c0 x1874670011490176/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826541.301185:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826541.301190:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826541.301194:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011490176:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826541.301213:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011490304:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826541.301235:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826541.301247:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826541/real 1787826541] req@ffff907d908a0f00 x1874670011490304/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826541.301259:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011490304/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826541.301268:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826541.301271:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826541.301274:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011490304:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826541.301279:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011490432:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826541.301298:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826541.301309:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826541/real 1787826541] req@ffff907d908a1680 x1874670011490432/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826541.301320:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011490432/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826541.301328:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826541.301332:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826541.301335:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011490432:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826541.301340:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011490560:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826541.301354:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826541.301361:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826541/real 1787826541] req@ffff907d908a12c0 x1874670011490560/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826541.301367:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011490560/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826541.301373:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826541.301375:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826541.301377:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011490560:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787826541.428808:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826542.452791:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826543.476862:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826544.500756:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826545.524810:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826546.420930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826546.420939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826546.420943:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826546.420952:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1950 00000100:00080000:1.0:1787826546.420962:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826546.420970:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826546.421008:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011490688/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826546.421035:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826546.421039:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826546.421041:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826546.421045:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1950 00000100:00080000:1.0:1787826546.421048:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826546.421052:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826546.421062:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826546.421075:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011490816/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826546.421085:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826546.421089:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1787826546.421089:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011490688:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787826546.421093:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826546.421100:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1950 00000100:00080000:1.0:1787826546.421104:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826546.421108:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826546.421113:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826546.421125:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011490944/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826546.421134:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826546.421137:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826546.421139:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826546.421143:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1950 00000100:00080000:1.0:1787826546.421147:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826546.421151:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826546.421156:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826546.421166:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011491072/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826546.421205:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011490816:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826546.421235:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011490944:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826546.421256:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011491072:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787826546.548920:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826547.572767:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826548.596788:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787826549.493051:0:2337:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:0.0:1787826549.493110:0:2337:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787826549.493129:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826549.493180:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826549.493190:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826549.493198:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:3.0:1787826549.493559:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826546/real 1787826549] req@ffff907d908a21c0 x1874670011490816/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826549.493588:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011490816/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826549.493601:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826549.494678:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826549.494696:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011490816:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826549.494709:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826546/real 1787826549] req@ffff907d908a2580 x1874670011490944/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826549.494733:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011490944/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826549.494750:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826549.494754:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826549.494758:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011490944:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826549.494765:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826546/real 1787826549] req@ffff907d908a3840 x1874670011491072/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826549.494778:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011491072/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826549.494786:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826549.494789:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826549.494792:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011491072:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826549.494799:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826546/real 1787826549] req@ffff907d908a2940 x1874670011490688/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826549.494812:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011490688/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826549.494820:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826549.494823:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826549.494827:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011490688:192.168.203.104@tcp:250:kworker.0 00010000:00010000:3.0:1787826549.620743:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826550.644803:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826551.540785:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826551.540796:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826551.540809:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826551.540818:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1955 00000100:00080000:0.0:1787826551.540829:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826551.540836:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826551.540869:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011491200/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826551.540901:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826551.540905:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826551.540907:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826551.540911:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1955 00000100:00080000:0.0:1787826551.540915:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826551.540918:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826551.540928:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826551.540950:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011491328/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826551.540962:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787826551.540963:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011491200:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787826551.540968:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826551.540972:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826551.540977:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1955 00000100:00080000:0.0:1787826551.540980:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826551.540984:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826551.540990:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826551.541002:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011491456/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826551.541011:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826551.541015:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826551.541017:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826551.541020:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1955 00000100:00080000:0.0:1787826551.541024:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826551.541030:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:3.0:1787826551.541031:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:0.0:1787826551.541037:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826551.541049:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011491584/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826551.541084:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826551/real 1787826551] req@ffff907d908452c0 x1874670011491200/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826551.541101:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011491200/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826551.541111:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826551.541125:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826551.541129:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011491200:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826551.541155:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011491328:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826551.541178:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826551.541190:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826551/real 1787826551] req@ffff907d90844f00 x1874670011491328/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826551.541214:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011491328/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826551.541221:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826551.541224:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826551.541227:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011491328:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826551.541241:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011491456:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826551.541261:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826551.541273:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826551/real 1787826551] req@ffff907d90846940 x1874670011491456/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826551.541284:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011491456/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826551.541297:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826551.541300:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826551.541303:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011491456:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826551.541309:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011491584:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826551.541325:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826551.541335:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826551/real 1787826551] req@ffff907d90847480 x1874670011491584/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826551.541346:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011491584/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826551.541353:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826551.541355:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826551.541359:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011491584:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826551.668844:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826552.692816:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826553.716831:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826554.740775:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826555.764748:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787826556.660896:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826556.660907:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787826556.660911:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826556.660920:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1960 00000100:00080000:3.0:1787826556.660930:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826556.660937:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826556.660981:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011491712/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826556.661009:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826556.661014:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787826556.661016:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826556.661019:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1960 00000100:00080000:3.0:1787826556.661023:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826556.661026:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826556.661035:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826556.661047:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011491840/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826556.661059:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011491712:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787826556.661059:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826556.661065:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787826556.661068:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826556.661072:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1960 00000100:00080000:3.0:1787826556.661076:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826556.661079:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826556.661083:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826556.661094:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011491968/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826556.661103:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826556.661106:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787826556.661108:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826556.661111:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1960 00000100:00080000:3.0:1787826556.661122:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826556.661126:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826556.661132:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787826556.661142:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826556.661143:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011492096/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826556.661210:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826556/real 1787826556] req@ffff907da562f840 x1874670011491712/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826556.661229:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011491712/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826556.661241:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826556.661246:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826556.661251:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011491712:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826556.661273:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011491840:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826556.661297:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826556.661310:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826556/real 1787826556] req@ffff907da562da40 x1874670011491840/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826556.661336:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011491840/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826556.661344:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826556.661346:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826556.661348:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011491840:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826556.661352:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011491968:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826556.661369:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826556.661376:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826556/real 1787826556] req@ffff907da562e1c0 x1874670011491968/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826556.661382:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011491968/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826556.661387:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826556.661389:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826556.661391:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011491968:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826556.661394:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011492096:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826556.661402:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826556.661407:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826556/real 1787826556] req@ffff907da562e940 x1874670011492096/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826556.661413:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011492096/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826556.661426:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826556.661428:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826556.661430:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011492096:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826556.788790:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826557.812744:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826558.836947:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826559.860797:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826560.884746:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787826561.780725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826561.780735:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787826561.780739:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826561.780748:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1965 00000100:00080000:3.0:1787826561.780759:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826561.780767:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826561.780816:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011492224/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826561.780859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826561.780863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787826561.780865:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826561.780875:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1965 00000100:00080000:3.0:1787826561.780879:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826561.780883:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826561.780894:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826561.780917:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011492224:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826561.780921:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011492352/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826561.780934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826561.780937:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787826561.780939:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826561.780943:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1965 00000100:00080000:3.0:1787826561.780947:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826561.780951:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826561.780957:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826561.780969:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011492480/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826561.780979:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826561.780982:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787826561.780984:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826561.780990:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1965 00000800:00000100:1.0:1787826561.780991:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787826561.780995:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826561.781001:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826561.781006:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826561.781018:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011492608/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826561.781041:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826561/real 1787826561] req@ffff907da562fc00 x1874670011492224/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826561.781059:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011492224/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826561.781068:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826561.781071:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826561.781075:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011492224:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826561.781097:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011492352:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826561.781121:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826561.781133:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826561/real 1787826561] req@ffff907da562f0c0 x1874670011492352/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826561.781142:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011492352/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826561.781147:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826561.781149:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826561.781151:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011492352:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826561.781155:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011492480:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826561.781170:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826561.781186:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826561/real 1787826561] req@ffff907da562c3c0 x1874670011492480/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826561.781201:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011492480/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826561.781208:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826561.781211:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826561.781214:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011492480:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826561.781223:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011492608:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826561.781243:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826561.781253:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826561/real 1787826561] req@ffff907da562d2c0 x1874670011492608/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826561.781263:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011492608/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826561.781270:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826561.781273:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826561.781277:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011492608:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826561.908906:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826562.932829:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826563.956755:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826564.980829:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826566.004796:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826566.900883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826566.900894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826566.900898:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826566.900907:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1970 00000100:00080000:0.0:1787826566.900917:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826566.900924:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826566.900959:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011492736/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826566.900990:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826566.900994:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826566.900996:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826566.901000:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1970 00000100:00080000:0.0:1787826566.901003:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826566.901007:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826566.901017:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826566.901029:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011492864/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826566.901040:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826566.901044:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787826566.901046:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011492736:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787826566.901047:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826566.901054:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1970 00000100:00080000:0.0:1787826566.901058:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826566.901062:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826566.901066:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826566.901077:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011492992/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826566.901085:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826566.901089:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826566.901091:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826566.901094:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1970 00000100:00080000:0.0:1787826566.901100:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826566.901104:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826566.901109:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787826566.901116:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787826566.901121:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011493120/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826566.901164:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826566/real 1787826566] req@ffff907d90845a40 x1874670011492736/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826566.901177:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011492736/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826566.901184:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826566.901187:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826566.901189:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011492736:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826566.901212:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011492864:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826566.901237:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826566.901261:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826566/real 1787826566] req@ffff907d90847480 x1874670011492864/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826566.901273:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011492864/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826566.901281:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826566.901284:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826566.901286:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011492864:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826566.901292:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011492992:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826566.901311:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826566.901322:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826566/real 1787826566] req@ffff907d90846940 x1874670011492992/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826566.901333:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011492992/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826566.901341:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826566.901343:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826566.901345:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011492992:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826566.901348:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011493120:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826566.901362:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826566.901373:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826566/real 1787826566] req@ffff907d90844f00 x1874670011493120/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826566.901380:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011493120/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826566.901384:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826566.901386:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826566.901388:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011493120:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787826567.028803:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826568.052799:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826569.076820:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826570.100774:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826571.124716:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826572.020803:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826572.020813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826572.020817:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826572.020826:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1975 00000100:00080000:2.0:1787826572.020837:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826572.020845:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826572.020882:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011493248/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826572.020910:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826572.020914:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826572.020916:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826572.020920:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1975 00000100:00080000:2.0:1787826572.020923:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826572.020927:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826572.020936:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826572.020947:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011493376/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826572.020956:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826572.020959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826572.020961:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826572.020964:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1975 00000100:00080000:2.0:1787826572.020967:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826572.020970:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826572.020974:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826572.020982:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011493504/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826572.020989:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826572.020992:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826572.020993:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826572.020996:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1975 00000100:00080000:2.0:1787826572.020998:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826572.021002:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826572.021005:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826572.021013:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011493632/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826572.021446:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011493248:192.168.203.104@tcp:250:kworker.0 00000800:00000100:1.0:1787826572.021506:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826572.021541:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826572/real 1787826572] req@ffff907d9114ad00 x1874670011493248/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826572.021556:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011493248/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826572.021567:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826572.021570:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826572.021574:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011493248:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826572.021580:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011493376:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826572.021599:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826572.021633:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826572/real 1787826572] req@ffff907d91148f00 x1874670011493376/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826572.021645:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011493376/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826572.021651:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826572.021655:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826572.021657:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011493376:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826572.021663:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011493504:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826572.021681:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826572.021690:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826572/real 1787826572] req@ffff907d9114a940 x1874670011493504/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826572.021700:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011493504/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826572.021707:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826572.021709:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826572.021712:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011493504:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826572.021717:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011493632:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826572.021730:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826572.021739:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826572/real 1787826572] req@ffff907d911492c0 x1874670011493632/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826572.021749:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011493632/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826572.021756:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826572.021758:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826572.021761:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011493632:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787826572.148781:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826573.173356:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826574.197736:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826575.221098:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826576.245185:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826577.140779:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826577.140794:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826577.140800:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826577.140813:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1980 00000100:00080000:0.0:1787826577.140826:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826577.140834:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826577.140882:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011493760/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826577.140910:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826577.140914:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826577.140916:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826577.140921:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1980 00000100:00080000:0.0:1787826577.140924:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826577.140928:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826577.140939:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826577.140952:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011493888/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826577.140962:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826577.140966:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826577.140967:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826577.140971:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1980 00000100:00080000:0.0:1787826577.140974:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826577.140978:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826577.140983:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826577.140992:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011494016/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826577.141001:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826577.141004:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826577.141006:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826577.141010:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1980 00000100:00080000:0.0:1787826577.141013:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826577.141017:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826577.141022:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826577.141033:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011494144/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826577.141070:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011493760:192.168.203.104@tcp:250:kworker.0 00000800:00000100:1.0:1787826577.141163:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826577.141209:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826577/real 1787826577] req@ffff907d908452c0 x1874670011493760/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826577.141237:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011493760/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826577.141247:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826577.141252:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826577.141256:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011493760:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826577.141263:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011493888:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826577.141290:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826577.141303:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826577/real 1787826577] req@ffff907d90846d00 x1874670011493888/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826577.141314:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011493888/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826577.141322:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826577.141325:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826577.141328:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011493888:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826577.141335:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011494016:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826577.141353:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826577.141376:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826577/real 1787826577] req@ffff907d90845680 x1874670011494016/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826577.141383:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011494016/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826577.141389:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826577.141391:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826577.141393:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011494016:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826577.141419:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011494144:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826577.141439:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826577.141449:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826577/real 1787826577] req@ffff907d90844780 x1874670011494144/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826577.141464:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011494144/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826577.141471:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826577.141473:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826577.141476:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011494144:192.168.203.104@tcp:8:kworker.0 00010000:00010000:0.0:1787826577.268811:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826578.292826:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826579.316765:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826580.340900:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826581.365239:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826582.261007:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826582.261015:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826582.261019:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826582.261029:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1986 00000100:00080000:0.0:1787826582.261040:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826582.261047:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826582.261081:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011494272/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826582.261140:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011494272:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787826582.261487:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826582.261492:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826582.261494:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826582.261498:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1986 00000100:00080000:0.0:1787826582.261503:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826582.261507:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826582.261521:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826582.261535:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011494400/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826582.261552:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826582.261555:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826582.261557:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826582.261561:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1986 00000100:00080000:0.0:1787826582.261565:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826582.261569:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826582.261575:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826582.261575:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011494400:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787826582.261590:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011494528/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826582.261599:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826582.261602:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826582.261604:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826582.261631:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1986 00000100:00100000:1.0:1787826582.261655:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011494528:192.168.203.104@tcp:8:kworker.0 00000100:00080000:0.0:1787826582.261763:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826582.261769:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826582.261774:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826582.261786:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011494656/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826582.262260:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011494656:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787826582.388740:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826583.412953:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826584.436800:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787826585.332877:0:2338:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:2.0:1787826585.332919:0:2338:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787826585.332935:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787826585.332980:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787826585.332994:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787826585.333001:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:1.0:1787826585.333177:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826582/real 1787826585] req@ffff907d90844000 x1874670011494656/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826585.333201:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011494656/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826585.333211:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826585.333215:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826585.333219:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011494656:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826585.333227:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826582/real 1787826585] req@ffff907d908443c0 x1874670011494528/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826585.333236:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011494528/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826585.333242:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826585.333246:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826585.333248:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011494528:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826585.333254:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826582/real 1787826585] req@ffff907d90844b40 x1874670011494400/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826585.333264:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011494400/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826585.333270:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826585.333273:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826585.333276:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011494400:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826585.333287:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826582/real 1787826585] req@ffff907d90845e00 x1874670011494272/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826585.333298:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011494272/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826585.333317:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826585.333321:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826585.333324:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011494272:192.168.203.104@tcp:250:kworker.0 00010000:00010000:0.0:1787826585.460758:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826586.484775:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787826587.381257:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826587.381267:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787826587.381271:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826587.381280:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1991 00000100:00080000:3.0:1787826587.381303:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826587.381310:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826587.381344:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011494784/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826587.381376:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826587.381379:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787826587.381381:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826587.381385:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1991 00000100:00080000:3.0:1787826587.381388:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826587.381391:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826587.381401:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826587.381412:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011494912/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826587.381420:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826587.381423:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787826587.381424:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826587.381428:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1991 00000100:00080000:3.0:1787826587.381430:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826587.381434:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826587.381438:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826587.381446:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011495040/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826587.381454:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826587.381457:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787826587.381458:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826587.381461:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1991 00000100:00080000:3.0:1787826587.381464:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826587.381468:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826587.381472:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826587.381480:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011495168/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826587.381564:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011494784:192.168.203.104@tcp:250:kworker.0 00000800:00000100:1.0:1787826587.381661:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826587.381696:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826587/real 1787826587] req@ffff907da562e580 x1874670011494784/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826587.381711:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011494784/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826587.381720:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826587.381723:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826587.381727:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011494784:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826587.381733:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011494912:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826587.381754:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826587.381763:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826587/real 1787826587] req@ffff907da562c000 x1874670011494912/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826587.381774:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011494912/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826587.381781:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826587.381784:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826587.381787:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011494912:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826587.381793:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011495040:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826587.381812:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826587.381823:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826587/real 1787826587] req@ffff907da562f480 x1874670011495040/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826587.381833:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011495040/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826587.381840:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826587.381843:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826587.381846:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011495040:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826587.381851:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011495168:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826587.381867:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826587.381877:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826587/real 1787826587] req@ffff907da562c780 x1874670011495168/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826587.381888:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011495168/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826587.381894:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826587.381897:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826587.381899:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011495168:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787826587.508940:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826588.532762:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826589.556793:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826590.580739:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826591.604934:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787826592.500729:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826592.500736:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787826592.500740:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826592.500748:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 1996 00000100:00080000:3.0:1787826592.500757:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826592.500764:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826592.500810:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011495296/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826592.500835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826592.500837:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787826592.500839:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826592.500841:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1996 00000100:00080000:3.0:1787826592.500844:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826592.500847:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826592.500855:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826592.500865:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011495424/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826592.500873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826592.500877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787826592.500878:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826592.500882:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1996 00000100:00080000:3.0:1787826592.500885:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826592.500889:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826592.500895:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826592.500912:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011495552/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826592.500923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826592.500926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787826592.500928:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826592.500934:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 1996 00000100:00080000:3.0:1787826592.500939:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787826592.500945:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011495296:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787826592.500946:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826592.500959:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826592.500978:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011495680/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787826592.501044:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826592.501110:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826592/real 1787826592] req@ffff907da562d680 x1874670011495296/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826592.501143:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011495296/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826592.501155:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826592.501159:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826592.501164:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011495296:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826592.501172:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011495424:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826592.501197:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826592.501211:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826592/real 1787826592] req@ffff907da562cf00 x1874670011495424/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826592.501222:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011495424/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826592.501231:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826592.501234:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826592.501238:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011495424:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826592.501281:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011495552:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826592.501304:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826592.501316:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826592/real 1787826592] req@ffff907da562ed00 x1874670011495552/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826592.501327:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011495552/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826592.501340:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826592.501343:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826592.501347:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011495552:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826592.501353:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011495680:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826592.501370:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826592.501380:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826592/real 1787826592] req@ffff907da562c780 x1874670011495680/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826592.501391:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011495680/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826592.501399:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826592.501402:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826592.501406:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011495680:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826592.628774:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826593.652744:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826594.676878:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826595.700751:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826596.724739:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826597.620758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826597.620767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826597.620771:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826597.620780:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2001 00000100:00080000:0.0:1787826597.620791:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826597.620798:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826597.620834:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011495808/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826597.620859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826597.620863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826597.620864:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826597.620868:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2001 00000100:00080000:0.0:1787826597.620872:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826597.620876:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826597.620886:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826597.620896:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011495808:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787826597.620901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011495936/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826597.620913:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826597.620916:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826597.620918:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826597.620922:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2001 00000100:00080000:0.0:1787826597.620925:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826597.620929:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826597.620933:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826597.620945:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011496064/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787826597.620950:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787826597.620961:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826597.620964:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826597.620967:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826597.620971:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2001 00000100:00080000:0.0:1787826597.620975:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826597.620978:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826597.620983:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826597.620992:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826597/real 1787826597] req@ffff907d908470c0 x1874670011495808/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787826597.620996:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908461c0 x1874670011496192/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826597.621022:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011495808/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826597.621034:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826597.621038:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826597.621042:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011495808:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826597.621062:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011495936:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826597.621080:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826597.621088:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826597/real 1787826597] req@ffff907d90847c00 x1874670011495936/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826597.621095:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011495936/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826597.621100:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826597.621102:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826597.621104:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011495936:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826597.621108:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011496064:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826597.621117:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826597.621123:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826597/real 1787826597] req@ffff907d90846580 x1874670011496064/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826597.621129:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011496064/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826597.621134:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826597.621136:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826597.621137:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011496064:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826597.621141:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011496192:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826597.621149:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826597.621154:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826597/real 1787826597] req@ffff907d908461c0 x1874670011496192/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826597.621161:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908461c0 x1874670011496192/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826597.621165:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826597.621167:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826597.621168:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011496192:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826597.748796:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826598.772872:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826599.796799:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826600.820782:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826601.844742:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826602.740734:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826602.740742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826602.740746:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826602.740753:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2006 00000100:00080000:1.0:1787826602.740762:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826602.740768:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826602.740806:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011496320/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826602.740832:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826602.740841:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826602.740843:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826602.740849:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2006 00000100:00080000:1.0:1787826602.740853:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826602.740857:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826602.740866:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826602.740878:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011496448/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826602.740887:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826602.740890:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826602.740892:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826602.740894:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2006 00000100:00080000:1.0:1787826602.740897:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826602.740899:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826602.740902:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826602.740909:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011496576/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826602.740914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826602.740917:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826602.740918:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826602.740919:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2006 00000100:00080000:1.0:1787826602.740921:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826602.740922:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826602.740924:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826602.740931:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011496704/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826602.741006:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011496320:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787826602.741065:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826602.741100:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826602/real 1787826602] req@ffff907d908a1e00 x1874670011496320/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826602.741111:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011496320/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826602.741118:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826602.741121:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826602.741124:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011496320:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826602.741128:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011496448:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826602.741152:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826602.741160:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826602/real 1787826602] req@ffff907d908a2d00 x1874670011496448/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826602.741167:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011496448/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826602.741171:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826602.741173:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826602.741175:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011496448:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826602.741178:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011496576:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826602.741205:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826602.741211:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826602/real 1787826602] req@ffff907d908a0000 x1874670011496576/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826602.741218:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011496576/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826602.741223:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826602.741225:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826602.741227:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011496576:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826602.741230:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011496704:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826602.741239:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826602.741244:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826602/real 1787826602] req@ffff907d908a2940 x1874670011496704/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826602.741250:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011496704/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826602.741254:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826602.741256:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826602.741257:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011496704:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826602.868892:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826603.892744:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826604.916942:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826605.940863:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826606.964825:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826607.860780:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826607.860793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826607.860798:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826607.860810:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2011 00000100:00080000:0.0:1787826607.860825:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826607.860833:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826607.860879:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847840 x1874670011496832/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826607.860915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826607.860918:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826607.860920:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826607.860925:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2011 00000100:00080000:0.0:1787826607.860928:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826607.860931:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826607.860946:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826607.860961:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908461c0 x1874670011496960/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826607.860971:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826607.860973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826607.860975:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826607.860979:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2011 00000100:00080000:0.0:1787826607.860982:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826607.860986:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826607.860990:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826607.860997:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011496832:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787826607.861005:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011497088/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826607.861015:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826607.861018:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826607.861020:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826607.861023:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2011 00000100:00080000:0.0:1787826607.861027:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826607.861031:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826607.861035:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826607.861045:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011497216/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787826607.861086:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826607.861159:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826607/real 1787826607] req@ffff907d90847840 x1874670011496832/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826607.861180:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847840 x1874670011496832/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826607.861193:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826607.861198:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826607.861202:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011496832:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826607.861210:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011496960:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826607.861233:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826607.861245:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826607/real 1787826607] req@ffff907d908461c0 x1874670011496960/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826607.861257:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908461c0 x1874670011496960/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826607.861264:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826607.861268:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826607.861271:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011496960:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826607.861296:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011497088:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826607.861316:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826607.861326:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826607/real 1787826607] req@ffff907d90846580 x1874670011497088/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826607.861337:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011497088/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826607.861345:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826607.861348:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826607.861351:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011497088:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826607.861357:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011497216:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826607.861374:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826607.861384:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826607/real 1787826607] req@ffff907d90847c00 x1874670011497216/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826607.861395:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011497216/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826607.861403:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826607.861406:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826607.861409:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011497216:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826607.988810:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826609.012880:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826610.036726:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826611.061218:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826612.084811:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826612.980745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826612.980761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826612.980767:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826612.980777:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2016 00000100:00080000:1.0:1787826612.980788:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826612.980794:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826612.980829:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011497344/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826612.980858:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826612.980862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826612.980863:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826612.980867:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2016 00000100:00080000:1.0:1787826612.980870:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826612.980873:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826612.980882:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826612.980891:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011497472/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826612.980899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826612.980902:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826612.980903:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826612.980906:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2016 00000100:00080000:1.0:1787826612.980910:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826612.980913:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826612.980916:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826612.980924:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011497600/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826612.980931:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826612.980934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826612.980935:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826612.980938:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2016 00000100:00080000:1.0:1787826612.980940:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826612.980943:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826612.980947:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826612.980954:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011497728/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826612.981039:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011497344:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787826612.981097:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826612.981153:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826612/real 1787826612] req@ffff907d908a3840 x1874670011497344/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826612.981170:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011497344/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826612.981180:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826612.981184:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826612.981188:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011497344:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826612.981194:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011497472:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826612.981213:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826612.981224:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826612/real 1787826612] req@ffff907d908a2580 x1874670011497472/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826612.981234:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011497472/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826612.981241:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826612.981244:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826612.981248:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011497472:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826612.981253:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011497600:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826612.981268:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826612.981276:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826612/real 1787826612] req@ffff907d908a21c0 x1874670011497600/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826612.981286:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011497600/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826612.981293:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826612.981296:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826612.981298:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011497600:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826612.981302:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011497728:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826612.981316:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826612.981324:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826612/real 1787826612] req@ffff907d908a12c0 x1874670011497728/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826612.981333:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011497728/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826612.981340:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826612.981343:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826612.981345:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011497728:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787826613.108784:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826614.132808:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826615.156915:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826616.180848:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826617.205655:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826618.100950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826618.100957:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826618.100960:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826618.100967:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2021 00000100:00080000:1.0:1787826618.100976:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826618.100981:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826618.101016:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011497856/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826618.101051:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826618.101055:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826618.101057:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826618.101061:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2021 00000100:00080000:1.0:1787826618.101065:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826618.101069:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826618.101091:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826618.101098:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011497856:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826618.101105:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011497984/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826618.101114:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826618.101116:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826618.101117:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826618.101120:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2021 00000100:00080000:1.0:1787826618.101125:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826618.101129:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826618.101134:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826618.101144:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a30c0 x1874670011498112/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826618.101153:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1787826618.101159:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787826618.101160:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826618.101164:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826618.101169:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2021 00000100:00080000:1.0:1787826618.101174:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826618.101177:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826618.101181:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826618.101192:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011498240/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826618.101211:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826618/real 1787826618] req@ffff907d908a1680 x1874670011497856/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826618.101229:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011497856/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826618.101239:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826618.101244:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826618.101248:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011497856:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826618.101273:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011497984:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826618.101293:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826618.101302:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826618/real 1787826618] req@ffff907d908a0f00 x1874670011497984/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826618.101320:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011497984/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826618.101332:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826618.101335:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826618.101339:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011497984:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826618.101346:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011498112:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826618.101372:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826618.101388:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826618/real 1787826618] req@ffff907d908a30c0 x1874670011498112/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826618.101413:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a30c0 x1874670011498112/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826618.101421:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826618.101425:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826618.101429:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011498112:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826618.101435:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011498240:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826618.101456:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826618.101466:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826618/real 1787826618] req@ffff907d908a12c0 x1874670011498240/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826618.101534:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011498240/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826618.101543:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826618.101559:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826618.101563:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011498240:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787826618.228801:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826619.252806:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826620.277147:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826621.300868:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826622.324879:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826623.220744:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826623.220755:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826623.220759:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826623.220769:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2026 00000100:00080000:1.0:1787826623.220781:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826623.220788:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826623.220824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011498368/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826623.220857:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826623.220861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826623.220863:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826623.220866:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2026 00000100:00080000:1.0:1787826623.220870:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826623.220873:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826623.220882:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826623.220893:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011498496/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826623.220908:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826623.220911:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826623.220912:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826623.220916:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2026 00000100:00080000:1.0:1787826623.220919:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826623.220922:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826623.220926:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826623.220935:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011498624/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826623.220942:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826623.220945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826623.220946:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826623.220949:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2026 00000100:00080000:1.0:1787826623.220952:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826623.220955:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826623.220958:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826623.220965:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011498752/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826623.221073:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011498368:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787826623.221143:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826623.221177:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826623/real 1787826623] req@ffff907d908a21c0 x1874670011498368/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826623.221193:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011498368/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826623.221204:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826623.221207:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826623.221211:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011498368:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826623.221217:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011498496:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826623.221237:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826623.221248:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826623/real 1787826623] req@ffff907d908a2580 x1874670011498496/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826623.221258:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011498496/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826623.221266:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826623.221268:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826623.221272:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011498496:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826623.221277:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011498624:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826623.221295:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826623.221306:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826623/real 1787826623] req@ffff907d908a3840 x1874670011498624/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826623.221323:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011498624/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826623.221331:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826623.221333:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826623.221336:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011498624:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826623.221341:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011498752:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826623.221358:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826623.221368:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826623/real 1787826623] req@ffff907d908a2940 x1874670011498752/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826623.221379:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011498752/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826623.221386:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826623.221389:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826623.221392:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011498752:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787826623.349008:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826624.372782:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826625.398234:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826626.420771:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826627.444778:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826628.340749:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826628.340759:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826628.340764:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826628.340772:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2032 00000100:00080000:0.0:1787826628.340781:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826628.340792:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826628.340838:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011498880/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826628.340865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826628.340868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826628.340869:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826628.340872:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2032 00000100:00080000:0.0:1787826628.340875:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826628.340877:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826628.340886:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826628.340895:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011499008/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826628.340901:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826628.340903:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826628.340904:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826628.340906:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2032 00000100:00080000:0.0:1787826628.340908:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826628.340910:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826628.340912:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826628.340917:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011499136/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826628.340922:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826628.340924:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826628.340925:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826628.340927:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2032 00000100:00080000:0.0:1787826628.340929:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826628.340930:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826628.340932:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826628.340937:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011499264/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826628.341072:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011498880:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787826628.341136:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826628.341171:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826628/real 1787826628] req@ffff907d908470c0 x1874670011498880/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826628.341183:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011498880/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826628.341193:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826628.341197:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826628.341200:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011498880:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826628.341212:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011499008:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826628.341234:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826628.341246:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826628/real 1787826628] req@ffff907d90845e00 x1874670011499008/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826628.341256:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011499008/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826628.341264:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826628.341267:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826628.341269:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011499008:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826628.341275:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011499136:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826628.341294:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826628.341304:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826628/real 1787826628] req@ffff907d90844b40 x1874670011499136/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826628.341311:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011499136/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826628.341316:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826628.341318:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826628.341319:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011499136:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826628.341324:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011499264:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826628.341334:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826628.341339:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826628/real 1787826628] req@ffff907d908443c0 x1874670011499264/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826628.341346:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011499264/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826628.341350:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826628.341351:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826628.341353:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011499264:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787826628.469900:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826629.494352:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826630.516877:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826631.540815:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826632.564770:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826633.461795:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826633.461804:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826633.461809:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826633.461819:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2037 00000100:00080000:0.0:1787826633.461830:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826633.461837:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826633.461876:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011499392/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826633.461906:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826633.461909:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826633.461911:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826633.461915:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2037 00000100:00080000:0.0:1787826633.461919:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826633.461923:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826633.461933:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826633.461947:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011499520/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826633.461953:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011499392:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787826633.461963:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826633.461980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826633.461981:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826633.461986:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2037 00000100:00080000:0.0:1787826633.461990:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826633.461994:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826633.461999:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826633.462010:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011499648/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826633.462019:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826633.462023:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826633.462024:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826633.462030:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2037 00000800:00000100:3.0:1787826633.462031:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787826633.462035:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826633.462040:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826633.462045:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826633.462056:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011499776/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826633.462078:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826633/real 1787826633] req@ffff907d90844000 x1874670011499392/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826633.462092:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011499392/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826633.462099:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826633.462103:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826633.462113:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011499392:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826633.462137:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011499520:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826633.462159:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826633.462172:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826633/real 1787826633] req@ffff907d90844780 x1874670011499520/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826633.462184:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011499520/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826633.462192:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826633.462195:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826633.462198:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011499520:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826633.462203:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011499648:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826633.462222:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826633.462234:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826633/real 1787826633] req@ffff907d90845680 x1874670011499648/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826633.462244:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011499648/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826633.462250:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826633.462251:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826633.462253:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011499648:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826633.462257:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011499776:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826633.462269:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826633.462275:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826633/real 1787826633] req@ffff907d90846d00 x1874670011499776/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826633.462282:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011499776/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826633.462286:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826633.462288:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826633.462290:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011499776:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826633.588776:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826634.612777:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826635.636840:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826636.661351:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826637.684956:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826638.581077:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826638.581087:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826638.581091:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826638.581101:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2042 00000100:00080000:1.0:1787826638.581112:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826638.581119:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826638.581152:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011499904/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826638.581181:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826638.581185:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826638.581186:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826638.581190:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2042 00000100:00080000:1.0:1787826638.581193:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826638.581197:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826638.581206:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826638.581219:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011500032/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826638.581228:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011499904:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787826638.581231:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826638.581236:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826638.581238:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826638.581242:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2042 00000100:00080000:1.0:1787826638.581246:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826638.581249:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826638.581254:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826638.581265:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011500160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826638.581283:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826638.581286:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826638.581288:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826638.581291:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2042 00000100:00080000:1.0:1787826638.581295:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826638.581299:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826638.581303:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787826638.581312:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826638.581317:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011500288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826638.581351:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826638/real 1787826638] req@ffff907d908a0000 x1874670011499904/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826638.581368:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011499904/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826638.581378:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826638.581382:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826638.581385:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011499904:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826638.581418:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011500032:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826638.581441:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826638.581454:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826638/real 1787826638] req@ffff907d908a2d00 x1874670011500032/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826638.581465:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011500032/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826638.581472:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826638.581476:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826638.581479:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011500032:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826638.581485:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011500160:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826638.581504:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826638.581515:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826638/real 1787826638] req@ffff907d908a1e00 x1874670011500160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826638.581524:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011500160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826638.581529:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826638.581532:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826638.581534:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011500160:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826638.581537:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011500288:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826638.581553:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826638.581562:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826638/real 1787826638] req@ffff907d908a2940 x1874670011500288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826638.581573:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011500288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826638.581580:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826638.581583:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826638.581586:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011500288:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826638.708786:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826639.732783:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826640.759071:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826641.781183:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826642.806979:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826643.700820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826643.700831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826643.700835:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826643.700845:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2047 00000100:00080000:0.0:1787826643.700857:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826643.700864:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826643.700914:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011500416/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826643.700941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826643.700945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826643.700947:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826643.700951:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2047 00000100:00080000:0.0:1787826643.700954:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826643.700958:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826643.700967:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826643.700983:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011500416:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787826643.700984:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011500544/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826643.701001:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826643.701003:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826643.701005:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826643.701009:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2047 00000100:00080000:0.0:1787826643.701012:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826643.701016:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826643.701021:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826643.701032:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011500672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787826643.701066:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787826643.701108:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826643.701112:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826643.701114:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826643.701118:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2047 00000100:00080000:0.0:1787826643.701122:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826643.701126:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826643.701131:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826643.701143:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011500800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826643.701253:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826643/real 1787826643] req@ffff907d908452c0 x1874670011500416/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826643.701272:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011500416/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826643.701282:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826643.701287:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826643.701301:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011500416:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826643.701323:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011500544:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826643.701348:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826643.701361:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826643/real 1787826643] req@ffff907d90844f00 x1874670011500544/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826643.701372:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011500544/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826643.701387:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826643.701390:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826643.701393:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011500544:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826643.701399:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011500672:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826643.701419:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826643.701430:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826643/real 1787826643] req@ffff907d90846940 x1874670011500672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826643.701441:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011500672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826643.701448:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826643.701451:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826643.701454:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011500672:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826643.701458:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011500800:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826643.701481:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826643.701494:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826643/real 1787826643] req@ffff907d90847480 x1874670011500800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826643.701505:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011500800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826643.701513:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826643.701516:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826643.701519:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011500800:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826643.828956:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826644.852857:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826645.876847:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826646.901932:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826647.924806:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826648.821983:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826648.821992:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826648.821996:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826648.822006:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2052 00000100:00080000:0.0:1787826648.822018:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826648.822025:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826648.822059:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011500928/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826648.822085:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826648.822089:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826648.822090:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826648.822095:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2052 00000100:00080000:0.0:1787826648.822098:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826648.822102:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826648.822112:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826648.822126:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011501056/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826648.822133:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011500928:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787826648.822140:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826648.822145:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826648.822147:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826648.822151:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2052 00000100:00080000:0.0:1787826648.822155:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826648.822159:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826648.822164:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826648.822177:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011501184/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826648.822186:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826648.822191:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826648.822193:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826648.822196:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2052 00000100:00080000:0.0:1787826648.822200:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826648.822204:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826648.822208:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826648.822220:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011501312/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826648.822243:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011501056:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826648.822284:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011501184:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826648.822308:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011501312:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787826648.948931:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826649.972834:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826650.997310:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787826651.893035:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:0.0:1787826651.893076:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787826651.893090:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826651.893128:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826651.893138:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826651.893147:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:3.0:1787826651.893168:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826648/real 1787826651] req@ffff907d90847480 x1874670011501056/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826651.893193:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011501056/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826651.893204:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826651.893208:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826651.893212:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011501056:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826651.893219:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826648/real 1787826651] req@ffff907d90846940 x1874670011501184/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826651.893229:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011501184/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826651.893237:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826651.893240:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826651.893243:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011501184:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826651.893249:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826648/real 1787826651] req@ffff907d90844f00 x1874670011501312/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826651.893261:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011501312/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826651.893268:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826651.893272:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826651.893275:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011501312:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826651.893281:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826648/real 1787826651] req@ffff907d90845a40 x1874670011500928/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826651.893293:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011500928/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826651.893301:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826651.893304:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826651.893307:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011500928:192.168.203.104@tcp:250:kworker.0 00010000:00010000:1.0:1787826652.020772:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826653.044823:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826653.940750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826653.940759:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826653.940763:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826653.940773:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2057 00000100:00080000:0.0:1787826653.940784:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826653.940792:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826653.940839:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011501440/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826653.940860:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826653.940864:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826653.940879:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826653.940889:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2057 00000100:00080000:0.0:1787826653.940893:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826653.940897:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826653.940907:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826653.940911:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011501440:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787826653.940924:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011501568/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826653.940940:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826653.940944:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826653.940946:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826653.940951:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2057 00000100:00080000:0.0:1787826653.940955:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826653.940960:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826653.940965:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826653.940978:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011501696/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826653.940988:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1787826653.940989:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787826653.940994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826653.940997:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826653.941004:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2057 00000100:00080000:0.0:1787826653.941009:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826653.941013:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826653.941018:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826653.941034:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011501824/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826653.941043:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826653/real 1787826653] req@ffff907d908452c0 x1874670011501440/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826653.941067:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011501440/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826653.941078:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826653.941083:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826653.941088:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011501440:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826653.941114:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011501568:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826653.941139:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826653.941158:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826653/real 1787826653] req@ffff907d90846d00 x1874670011501568/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826653.941170:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011501568/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826653.941178:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826653.941185:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826653.941189:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011501568:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826653.941195:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011501696:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826653.941223:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826653.941238:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826653/real 1787826653] req@ffff907d90845680 x1874670011501696/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826653.941249:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011501696/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826653.941258:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826653.941263:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826653.941268:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011501696:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826653.941274:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011501824:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826653.941292:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826653.941313:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826653/real 1787826653] req@ffff907d90844780 x1874670011501824/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826653.941335:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011501824/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826653.941347:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826653.941350:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826653.941353:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011501824:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787826654.068867:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826655.092843:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826656.116854:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826657.141212:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826658.164748:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826659.060726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826659.060736:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826659.060741:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826659.060750:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2062 00000100:00080000:1.0:1787826659.060761:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826659.060768:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826659.060812:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011501952/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826659.060844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826659.060848:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826659.060850:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826659.060853:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2062 00000100:00080000:1.0:1787826659.060857:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826659.060861:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826659.060870:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826659.060882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011502080/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826659.060891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826659.060894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826659.060896:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826659.060899:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2062 00000100:00080000:1.0:1787826659.060903:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826659.060906:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826659.060911:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826659.060920:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011502208/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826659.060929:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826659.060932:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826659.060933:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826659.060937:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2062 00000100:00080000:1.0:1787826659.060941:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826659.060945:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826659.060949:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826659.060959:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011502336/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826659.061117:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011501952:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787826659.061189:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826659.061225:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826659/real 1787826659] req@ffff907d908a3840 x1874670011501952/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826659.061241:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011501952/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826659.061251:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826659.061255:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826659.061258:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011501952:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826659.061265:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011502080:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826659.061286:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826659.061298:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826659/real 1787826659] req@ffff907d908a2580 x1874670011502080/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826659.061309:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011502080/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826659.061316:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826659.061320:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826659.061323:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011502080:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826659.061329:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011502208:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826659.061348:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826659.061359:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826659/real 1787826659] req@ffff907d908a21c0 x1874670011502208/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826659.061371:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011502208/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826659.061378:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826659.061381:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826659.061384:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011502208:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826659.061390:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011502336:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826659.061409:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826659.061420:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826659/real 1787826659] req@ffff907d908a12c0 x1874670011502336/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826659.061431:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011502336/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826659.061439:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826659.061442:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826659.061446:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011502336:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787826659.188801:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826660.213923:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826661.236805:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826662.260764:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826663.284792:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826664.180705:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826664.180712:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826664.180715:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826664.180723:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2067 00000100:00080000:1.0:1787826664.180732:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826664.180739:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826664.180788:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a30c0 x1874670011502464/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826664.180820:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826664.180823:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826664.180825:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826664.180829:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2067 00000100:00080000:1.0:1787826664.180832:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826664.180836:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826664.180846:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826664.180858:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011502592/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826664.180867:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826664.180870:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826664.180871:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826664.180874:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2067 00000100:00080000:1.0:1787826664.180878:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826664.180881:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826664.180885:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826664.180896:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011502720/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826664.180904:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826664.180907:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826664.180909:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826664.180913:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2067 00000100:00080000:1.0:1787826664.180916:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826664.180919:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826664.180924:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826664.180932:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011502848/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826664.181009:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011502464:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787826664.181063:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826664.181093:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826664/real 1787826664] req@ffff907d908a30c0 x1874670011502464/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826664.181103:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a30c0 x1874670011502464/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826664.181110:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826664.181113:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826664.181117:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011502464:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826664.181125:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011502592:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826664.181159:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826664.181170:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826664/real 1787826664] req@ffff907d908a0f00 x1874670011502592/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826664.181179:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011502592/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826664.181187:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826664.181190:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826664.181193:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011502592:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826664.181199:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011502720:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826664.181219:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826664.181228:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826664/real 1787826664] req@ffff907d908a1680 x1874670011502720/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826664.181239:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011502720/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826664.181246:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826664.181249:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826664.181251:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011502720:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826664.181256:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011502848:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826664.181281:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826664.181290:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826664/real 1787826664] req@ffff907d908a12c0 x1874670011502848/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826664.181300:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011502848/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826664.181307:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826664.181310:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826664.181313:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011502848:192.168.203.104@tcp:8:kworker.0 00010000:00010000:0.0:1787826664.308771:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826665.334905:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826666.356996:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826667.380843:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826668.406330:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826669.301175:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826669.301184:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826669.301187:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826669.301196:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2073 00000100:00080000:1.0:1787826669.301206:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826669.301211:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826669.301244:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011502976/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826669.301273:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826669.301277:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826669.301278:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826669.301282:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2073 00000100:00080000:1.0:1787826669.301286:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826669.301289:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826669.301299:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826669.301310:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011503104/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826669.301318:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826669.301321:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826669.301323:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826669.301326:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2073 00000100:00080000:1.0:1787826669.301330:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826669.301333:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826669.301337:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826669.301346:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011503232/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826669.301360:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826669.301363:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826669.301364:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826669.301368:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2073 00000100:00080000:1.0:1787826669.301370:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826669.301374:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826669.301378:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826669.301386:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011503360/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826669.301470:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011502976:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787826669.301532:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826669.301567:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826669/real 1787826669] req@ffff907d908a21c0 x1874670011502976/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826669.301583:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011502976/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826669.301592:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826669.301596:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826669.301600:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011502976:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826669.301627:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011503104:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826669.301650:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826669.301662:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826669/real 1787826669] req@ffff907d908a2580 x1874670011503104/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826669.301673:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011503104/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826669.301680:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826669.301682:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826669.301686:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011503104:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826669.301691:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011503232:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826669.301710:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826669.301729:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826669/real 1787826669] req@ffff907d908a3840 x1874670011503232/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826669.301741:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011503232/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826669.301749:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826669.301751:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826669.301754:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011503232:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826669.301758:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011503360:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826669.301776:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826669.301787:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826669/real 1787826669] req@ffff907d908a2940 x1874670011503360/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826669.301810:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011503360/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826669.301817:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826669.301820:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826669.301823:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011503360:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787826669.428835:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826670.452846:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826671.476830:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826672.500754:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826673.524978:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826674.420739:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826674.420752:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826674.420756:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826674.420766:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2078 00000100:00080000:1.0:1787826674.420777:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826674.420785:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826674.420834:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011503488/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826674.420865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826674.420868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826674.420870:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826674.420874:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2078 00000100:00080000:1.0:1787826674.420878:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826674.420886:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826674.420896:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826674.420909:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011503616/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826674.420921:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826674.420924:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826674.420926:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826674.420930:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2078 00000100:00080000:1.0:1787826674.420933:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826674.420937:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826674.420942:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826674.420960:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011503488:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826674.420961:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011503744/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826674.420984:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826674.420987:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826674.420989:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826674.420994:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2078 00000100:00080000:1.0:1787826674.420997:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826674.421001:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826674.421006:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826674.421019:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011503872/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787826674.421055:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826674.421114:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826674/real 1787826674] req@ffff907d908a1e00 x1874670011503488/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826674.421134:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011503488/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826674.421145:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826674.421149:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826674.421153:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011503488:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826674.421161:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011503616:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826674.421188:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826674.421200:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826674/real 1787826674] req@ffff907d908a2d00 x1874670011503616/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826674.421208:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011503616/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826674.421213:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826674.421215:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826674.421217:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011503616:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826674.421241:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011503744:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826674.421255:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826674.421262:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826674/real 1787826674] req@ffff907d908a0000 x1874670011503744/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826674.421269:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011503744/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826674.421274:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826674.421276:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826674.421278:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011503744:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826674.421281:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011503872:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826674.421289:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826674.421294:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826674/real 1787826674] req@ffff907d908a2940 x1874670011503872/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826674.421305:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011503872/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826674.421319:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826674.421322:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826674.421325:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011503872:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826674.549742:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826675.572819:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826676.596829:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826677.620837:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826678.644831:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826679.540770:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826679.540780:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826679.540796:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826679.540815:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2083 00000100:00080000:1.0:1787826679.540826:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826679.540833:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826679.540870:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011504000/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826679.540903:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826679.540906:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826679.540908:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826679.540912:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2083 00000100:00080000:1.0:1787826679.540916:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826679.540920:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826679.540934:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826679.540943:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011504128/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826679.540950:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826679.540953:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826679.540955:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826679.540958:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2083 00000100:00080000:1.0:1787826679.540961:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826679.540964:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826679.540969:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826679.540979:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011504256/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826679.540988:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826679.540991:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826679.540993:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826679.540996:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2083 00000100:00080000:1.0:1787826679.540999:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826679.541010:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826679.541015:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826679.541025:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011504384/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826679.541170:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011504000:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787826679.541241:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826679.541272:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826679/real 1787826679] req@ffff907d908a3840 x1874670011504000/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826679.541284:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011504000/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826679.541291:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826679.541293:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826679.541296:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011504000:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826679.541300:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011504128:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826679.541328:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826679.541340:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826679/real 1787826679] req@ffff907d908a2580 x1874670011504128/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826679.541358:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011504128/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826679.541364:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826679.541366:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826679.541368:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011504128:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826679.541372:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011504256:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826679.541386:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826679.541393:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826679/real 1787826679] req@ffff907d908a21c0 x1874670011504256/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826679.541400:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011504256/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826679.541404:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826679.541405:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826679.541407:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011504256:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826679.541410:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011504384:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826679.541419:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826679.541424:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826679/real 1787826679] req@ffff907d908a12c0 x1874670011504384/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826679.541430:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011504384/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826679.541438:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826679.541440:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826679.541443:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011504384:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826679.668835:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826680.692787:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826681.716806:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826682.740841:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826683.764811:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826684.660718:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826684.660726:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826684.660729:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826684.660736:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2088 00000100:00080000:1.0:1787826684.660745:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826684.660751:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826684.660779:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011504512/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826684.660801:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826684.660805:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826684.660807:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826684.660811:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2088 00000100:00080000:1.0:1787826684.660815:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826684.660820:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826684.660831:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826684.660844:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011504640/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826684.660851:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011504512:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787826684.660856:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826684.660860:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826684.660861:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826684.660864:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2088 00000100:00080000:1.0:1787826684.660867:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826684.660870:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826684.660873:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826684.660882:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a30c0 x1874670011504768/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826684.660888:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826684.660890:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826684.660891:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826684.660893:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2088 00000100:00080000:1.0:1787826684.660896:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826684.660898:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826684.660900:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787826684.660906:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826684.660910:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011504896/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826684.660944:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826684/real 1787826684] req@ffff907d908a1680 x1874670011504512/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826684.660956:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011504512/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826684.660963:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826684.660966:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826684.660969:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011504512:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826684.660985:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011504640:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826684.660999:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826684.661006:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826684/real 1787826684] req@ffff907d908a0f00 x1874670011504640/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826684.661013:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011504640/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826684.661018:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826684.661020:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826684.661022:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011504640:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826684.661025:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011504768:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826684.661035:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826684.661040:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826684/real 1787826684] req@ffff907d908a30c0 x1874670011504768/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826684.661047:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a30c0 x1874670011504768/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826684.661051:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826684.661052:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826684.661054:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a30c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011504768:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826684.661057:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011504896:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826684.661065:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826684.661070:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826684/real 1787826684] req@ffff907d908a12c0 x1874670011504896/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826684.661076:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011504896/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826684.661080:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826684.661082:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826684.661083:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011504896:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826684.788840:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826685.812794:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826686.836820:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826687.860847:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826688.884755:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826689.780733:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826689.780740:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826689.780743:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826689.780751:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2093 00000100:00080000:2.0:1787826689.780761:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826689.780767:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826689.780801:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011505024/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826689.780826:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826689.780829:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826689.780830:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826689.780833:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2093 00000100:00080000:2.0:1787826689.780836:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826689.780839:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826689.780846:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826689.780856:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011505152/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826689.780864:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826689.780867:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826689.780869:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787826689.780870:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011505024:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787826689.780872:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2093 00000100:00080000:2.0:1787826689.780877:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826689.780881:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826689.780885:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826689.780895:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011505280/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826689.780903:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826689.780907:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826689.780909:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826689.780913:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2093 00000100:00080000:2.0:1787826689.780918:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826689.780922:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826689.780927:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826689.780941:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011505408/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787826689.780944:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826689.780993:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826689/real 1787826689] req@ffff907d91148b40 x1874670011505024/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826689.781012:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011505024/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826689.781022:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826689.781027:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826689.781031:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011505024:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826689.781064:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011505152:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826689.781089:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826689.781104:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826689/real 1787826689] req@ffff907d911492c0 x1874670011505152/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826689.781116:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011505152/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826689.781124:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826689.781127:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826689.781131:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011505152:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826689.781140:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011505280:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826689.781161:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826689.781172:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826689/real 1787826689] req@ffff907d9114a940 x1874670011505280/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826689.781183:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011505280/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826689.781191:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826689.781195:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826689.781199:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011505280:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826689.781205:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011505408:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826689.781226:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826689.781239:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826689/real 1787826689] req@ffff907d91148f00 x1874670011505408/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826689.781262:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011505408/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826689.781271:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826689.781275:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826689.781280:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011505408:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787826689.908731:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826690.932751:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826691.956807:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826692.980778:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826694.004765:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826694.900708:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826694.900717:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826694.900721:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826694.900730:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2098 00000100:00080000:2.0:1787826694.900740:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826694.900747:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826694.900780:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011505536/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826694.900807:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826694.900811:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826694.900812:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826694.900816:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2098 00000100:00080000:2.0:1787826694.900819:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826694.900824:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826694.900833:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826694.900845:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011505664/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826694.900848:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011505536:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787826694.900866:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826694.900869:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826694.900871:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826694.900875:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2098 00000100:00080000:2.0:1787826694.900878:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826694.900881:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826694.900885:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826694.900895:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011505792/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826694.900903:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826694.900906:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826694.900908:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826694.900911:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2098 00000100:00080000:2.0:1787826694.900914:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826694.900916:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826694.900920:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787826694.900923:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787826694.900932:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011505920/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826694.900963:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826694/real 1787826694] req@ffff907d9114ad00 x1874670011505536/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826694.900981:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011505536/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826694.900992:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826694.900995:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826694.901000:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011505536:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826694.901019:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011505664:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826694.901040:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826694.901052:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826694/real 1787826694] req@ffff907d91149a40 x1874670011505664/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826694.901063:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011505664/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826694.901071:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826694.901074:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826694.901077:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011505664:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826694.901083:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011505792:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826694.901103:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826694.901114:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826694/real 1787826694] req@ffff907d9114a580 x1874670011505792/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826694.901131:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011505792/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826694.901138:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826694.901141:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826694.901144:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011505792:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826694.901150:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011505920:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826694.901167:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826694.901192:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826694/real 1787826694] req@ffff907d9114b840 x1874670011505920/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826694.901204:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011505920/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826694.901216:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826694.901219:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826694.901222:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011505920:192.168.203.104@tcp:8:kworker.0 00010000:00010000:0.0:1787826695.028735:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826696.052786:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826697.076747:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826698.100947:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826699.124808:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787826700.020764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826700.020771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787826700.020775:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826700.020782:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2103 00000100:00080000:3.0:1787826700.020792:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826700.020797:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826700.020838:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011506048/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826700.020864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826700.020866:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787826700.020867:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826700.020870:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2103 00000100:00080000:3.0:1787826700.020872:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826700.020874:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826700.020882:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826700.020891:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011506176/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826700.020900:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826700.020903:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787826700.020907:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787826700.020908:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011506048:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787826700.020918:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2103 00000100:00080000:3.0:1787826700.020922:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826700.020927:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826700.020932:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826700.020945:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011506304/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826700.020954:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826700.020957:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787826700.020959:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787826700.020964:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787826700.020967:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2103 00000100:00080000:3.0:1787826700.020972:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826700.020981:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826700.020987:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826700.020998:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011506432/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826700.021002:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826700/real 1787826700] req@ffff907da562f480 x1874670011506048/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826700.021025:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011506048/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826700.021033:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826700.021036:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826700.021039:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011506048:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826700.021057:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011506176:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826700.021071:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826700.021078:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826700/real 1787826700] req@ffff907da562c000 x1874670011506176/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826700.021086:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011506176/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826700.021090:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826700.021092:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826700.021094:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011506176:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826700.021097:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011506304:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826700.021107:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826700.021112:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826700/real 1787826700] req@ffff907da562e580 x1874670011506304/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826700.021118:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011506304/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826700.021123:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826700.021125:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826700.021126:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011506304:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826700.021130:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011506432:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826700.021138:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826700.021143:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826700/real 1787826700] req@ffff907da562d2c0 x1874670011506432/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826700.021149:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011506432/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826700.021153:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826700.021155:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826700.021161:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011506432:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787826700.148812:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826701.172773:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826702.196772:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826703.220842:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826704.244766:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787826705.140727:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826705.140736:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787826705.140740:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826705.140749:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2108 00000100:00080000:3.0:1787826705.140759:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826705.140766:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826705.140809:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011506560/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826705.140840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826705.140844:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787826705.140847:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826705.140851:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2108 00000100:00080000:3.0:1787826705.140855:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826705.140858:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826705.140867:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826705.140885:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011506688/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826705.140895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826705.140898:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787826705.140899:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826705.140902:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2108 00000100:00080000:3.0:1787826705.140905:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826705.140910:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826705.140913:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826705.140922:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011506816/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826705.140929:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826705.140932:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787826705.140933:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826705.140935:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2108 00000100:00080000:3.0:1787826705.140938:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826705.140940:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826705.140944:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826705.140952:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011506944/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826705.141037:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011506560:192.168.203.104@tcp:250:kworker.0 00000800:00000100:1.0:1787826705.141096:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826705.141146:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826705/real 1787826705] req@ffff907da562c3c0 x1874670011506560/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826705.141160:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011506560/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826705.141177:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826705.141181:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826705.141184:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011506560:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826705.141191:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011506688:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826705.141216:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826705.141229:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826705/real 1787826705] req@ffff907da562f0c0 x1874670011506688/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826705.141239:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011506688/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826705.141246:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826705.141249:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826705.141252:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011506688:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826705.141258:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011506816:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826705.141277:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826705.141288:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826705/real 1787826705] req@ffff907da562fc00 x1874670011506816/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826705.141298:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011506816/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826705.141305:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826705.141307:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826705.141310:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011506816:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826705.141315:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011506944:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826705.141332:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826705.141343:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826705/real 1787826705] req@ffff907da562e940 x1874670011506944/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826705.141359:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011506944/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826705.141366:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826705.141369:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826705.141372:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011506944:192.168.203.104@tcp:8:kworker.0 00010000:00010000:0.0:1787826705.268747:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826706.293689:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826707.316810:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826708.340835:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826709.364803:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787826710.260902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826710.260912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787826710.260916:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826710.260925:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2114 00000100:00080000:3.0:1787826710.260935:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826710.260942:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826710.260976:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011507072/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826710.260997:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826710.261000:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787826710.261001:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826710.261004:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2114 00000100:00080000:3.0:1787826710.261007:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826710.261009:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826710.261016:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826710.261024:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011507200/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826710.261030:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826710.261032:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787826710.261033:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826710.261036:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2114 00000100:00080000:3.0:1787826710.261038:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787826710.261045:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011507072:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787826710.261047:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826710.261053:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826710.261060:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011507328/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826710.261066:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826710.261068:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787826710.261069:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826710.261071:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2114 00000100:00080000:3.0:1787826710.261073:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826710.261075:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826710.261078:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826710.261087:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011507456/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787826710.261115:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826710.261154:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826710/real 1787826710] req@ffff907da562e1c0 x1874670011507072/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826710.261169:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011507072/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826710.261176:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826710.261179:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826710.261181:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011507072:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826710.261186:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011507200:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826710.261203:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826710.261214:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826710/real 1787826710] req@ffff907da562da40 x1874670011507200/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826710.261226:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011507200/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826710.261234:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826710.261237:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826710.261240:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011507200:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826710.261267:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011507328:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826710.261294:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826710.261304:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826710/real 1787826710] req@ffff907da562f840 x1874670011507328/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826710.261314:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011507328/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826710.261321:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826710.261324:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826710.261327:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011507328:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826710.261333:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011507456:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826710.261349:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826710.261360:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826710/real 1787826710] req@ffff907da562de00 x1874670011507456/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826710.261370:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011507456/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826710.261378:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826710.261380:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826710.261383:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011507456:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787826710.388838:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826711.412761:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826712.436771:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826713.460754:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826714.484821:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826715.380742:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826715.380751:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826715.380756:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826715.380765:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2119 00000100:00080000:2.0:1787826715.380775:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826715.380781:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826715.380813:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011507584/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826715.380848:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826715.380851:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826715.380853:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826715.380856:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2119 00000100:00080000:2.0:1787826715.380859:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826715.380863:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826715.380872:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826715.380882:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011507712/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826715.380890:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826715.380893:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826715.380895:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826715.380898:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2119 00000100:00080000:2.0:1787826715.380900:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826715.380903:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826715.380907:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826715.380915:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011507840/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826715.380922:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826715.380924:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826715.380926:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826715.380929:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2119 00000100:00080000:2.0:1787826715.380931:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826715.380934:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826715.380938:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826715.380946:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011507968/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826715.381030:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011507584:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826715.381135:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011507712:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826715.381165:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011507840:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826715.381187:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011507968:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826715.508807:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826716.532872:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826717.556810:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787826718.453065:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787826718.453291:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787826718.453307:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787826718.453344:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787826718.453352:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787826718.453360:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:3.0:1787826718.453572:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826715/real 1787826718] req@ffff907d91149680 x1874670011507584/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826718.454695:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011507584/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826718.454717:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826718.454723:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826718.454729:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011507584:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826718.454742:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826715/real 1787826718] req@ffff907d9114bc00 x1874670011507712/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826718.454765:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011507712/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826718.454771:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826718.454775:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826718.454778:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011507712:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826718.454784:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826715/real 1787826718] req@ffff907d911483c0 x1874670011507840/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826718.454796:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011507840/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826718.454804:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826718.454808:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826718.454811:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011507840:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826718.454819:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826715/real 1787826718] req@ffff907d9114b480 x1874670011507968/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826718.454830:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011507968/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826718.454837:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826718.454840:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826718.454843:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011507968:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787826718.580846:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826719.604740:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826720.500917:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826720.500931:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826720.500937:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826720.500948:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2124 00000100:00080000:2.0:1787826720.500961:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826720.500969:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826720.501014:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011508096/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826720.501052:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826720.501056:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826720.501058:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826720.501062:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2124 00000100:00080000:2.0:1787826720.501066:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826720.501069:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826720.501083:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826720.501096:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011508224/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826720.501106:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826720.501112:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826720.501114:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826720.501117:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2124 00000100:00080000:2.0:1787826720.501120:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826720.501124:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826720.501129:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826720.501139:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011508352/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826720.501149:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826720.501152:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826720.501154:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826720.501158:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2124 00000100:00080000:2.0:1787826720.501161:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826720.501164:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826720.501168:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826720.501176:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011508480/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826720.501288:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011508096:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787826720.501376:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826720.501428:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826720/real 1787826720] req@ffff907d9114b480 x1874670011508096/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826720.501448:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011508096/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826720.501460:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826720.501464:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826720.501469:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011508096:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826720.501477:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011508224:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826720.501510:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826720.501523:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826720/real 1787826720] req@ffff907d911483c0 x1874670011508224/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826720.501534:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011508224/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826720.501541:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826720.501544:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826720.501547:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011508224:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826720.501553:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011508352:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826720.501574:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826720.501586:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826720/real 1787826720] req@ffff907d9114bc00 x1874670011508352/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826720.501598:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011508352/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826720.502572:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826720.502583:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826720.502590:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011508352:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826720.502601:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011508480:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826720.502692:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826720.502712:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826720/real 1787826720] req@ffff907d91149680 x1874670011508480/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826720.502729:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011508480/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826720.502738:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826720.502742:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826720.502745:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011508480:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826720.628799:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826721.653108:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826722.676736:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826723.700763:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826724.724833:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826725.620717:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826725.620726:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826725.620730:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826725.620740:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2129 00000100:00080000:2.0:1787826725.620750:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826725.620758:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826725.620791:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011508608/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826725.620817:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826725.620820:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826725.620822:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826725.620826:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2129 00000100:00080000:2.0:1787826725.620830:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826725.620835:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826725.620844:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826725.620848:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011508608:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787826725.620860:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011508736/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826725.620871:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826725.620874:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826725.620876:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826725.620880:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2129 00000100:00080000:2.0:1787826725.620884:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826725.620888:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826725.620892:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826725.620902:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011508864/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787826725.620912:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787826725.620914:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826725.620918:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826725.620920:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826725.620925:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2129 00000100:00080000:2.0:1787826725.620935:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826725.620939:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826725.620944:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826725.620951:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826725/real 1787826725] req@ffff907d9114b840 x1874670011508608/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787826725.620956:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011508992/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826725.620984:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011508608/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826725.620995:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826725.621011:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826725.621015:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011508608:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826725.621036:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011508736:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826725.621057:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826725.621068:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826725/real 1787826725] req@ffff907d9114a580 x1874670011508736/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826725.621080:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011508736/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826725.621089:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826725.621092:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826725.621096:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011508736:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826725.621103:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011508864:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826725.621124:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826725.621135:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826725/real 1787826725] req@ffff907d91149a40 x1874670011508864/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826725.621146:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011508864/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826725.621153:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826725.621156:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826725.621159:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011508864:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826725.621164:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011508992:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826725.621182:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826725.621192:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826725/real 1787826725] req@ffff907d9114ad00 x1874670011508992/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826725.621202:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011508992/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826725.621208:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826725.621211:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826725.621213:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011508992:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826725.749059:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826726.772787:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826727.796788:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826728.820778:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826729.844793:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826730.740718:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826730.740729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826730.740733:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826730.740742:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2134 00000100:00080000:0.0:1787826730.740755:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826730.740763:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826730.740808:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011509120/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826730.740831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826730.740834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826730.740836:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826730.740839:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2134 00000100:00080000:0.0:1787826730.740842:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826730.740846:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826730.740856:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826730.740866:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011509248/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826730.740874:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826730.740876:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826730.740877:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826730.740880:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2134 00000100:00080000:0.0:1787826730.740883:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826730.740889:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826730.740891:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011509120:192.168.203.104@tcp:250:kworker.0 00000008:00100000:0.0:1787826730.740893:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826730.740907:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011509376/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826730.740917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826730.740920:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826730.740923:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826730.740928:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2134 00000100:00080000:0.0:1787826730.740932:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826730.740937:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826730.740943:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826730.740955:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011509504/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787826730.740978:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826730.741023:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826730/real 1787826730] req@ffff907d90844000 x1874670011509120/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826730.741043:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011509120/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826730.741054:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826730.741059:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826730.741063:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011509120:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826730.741086:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011509248:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826730.741159:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826730.741172:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826730/real 1787826730] req@ffff907d908443c0 x1874670011509248/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826730.741185:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011509248/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826730.741192:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826730.741195:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826730.741199:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011509248:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826730.741205:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011509376:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826730.741228:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826730.741238:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826730/real 1787826730] req@ffff907d90844b40 x1874670011509376/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826730.741249:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011509376/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826730.741256:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826730.741259:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826730.741262:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011509376:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826730.741268:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011509504:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826730.741286:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826730.741306:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826730/real 1787826730] req@ffff907d90845e00 x1874670011509504/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826730.741317:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011509504/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826730.741329:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826730.741333:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826730.741336:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011509504:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787826730.868862:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826731.892813:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826732.916828:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826733.940950:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826734.964719:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826735.860836:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826735.860846:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826735.860850:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826735.860860:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2139 00000100:00080000:1.0:1787826735.860870:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826735.860876:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826735.860908:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011509632/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826735.860939:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826735.860943:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826735.860945:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826735.860948:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2139 00000100:00080000:1.0:1787826735.860951:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826735.860954:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826735.860963:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826735.860973:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011509760/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826735.860982:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826735.860985:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826735.860987:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826735.860991:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2139 00000100:00080000:1.0:1787826735.860993:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826735.860996:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826735.861000:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826735.861007:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011509888/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826735.861015:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826735.861023:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826735.861025:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826735.861028:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2139 00000100:00080000:1.0:1787826735.861031:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826735.861034:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826735.861038:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826735.861046:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011510016/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826735.861495:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011509632:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787826735.861550:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826735.861597:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826735/real 1787826735] req@ffff907d908a21c0 x1874670011509632/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826735.861641:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011509632/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826735.861649:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826735.861653:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826735.861656:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011509632:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826735.861663:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011509760:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826735.861682:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826735.861692:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826735/real 1787826735] req@ffff907d908a2580 x1874670011509760/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826735.861701:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011509760/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826735.861708:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826735.861710:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826735.861713:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011509760:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826735.861717:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011509888:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826735.861732:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826735.861739:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826735/real 1787826735] req@ffff907d908a3840 x1874670011509888/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826735.861748:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011509888/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826735.861759:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826735.861762:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826735.861764:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011509888:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826735.861769:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011510016:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826735.861784:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826735.861854:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826735/real 1787826735] req@ffff907d908a2940 x1874670011510016/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826735.861865:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011510016/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826735.861871:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826735.861874:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826735.861877:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011510016:192.168.203.104@tcp:8:kworker.0 00010000:00010000:0.0:1787826735.988767:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826737.016355:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826738.036813:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826739.060852:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826740.084887:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826740.980807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826740.980818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826740.980822:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826740.980834:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2144 00000100:00080000:0.0:1787826740.980856:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826740.980864:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826740.980901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011510144/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826740.980934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826740.980938:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826740.980940:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826740.980943:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2144 00000100:00080000:0.0:1787826740.980947:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826740.980951:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826740.980961:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826740.980974:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011510272/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826740.980985:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826740.980988:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826740.980991:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826740.980995:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2144 00000100:00080000:0.0:1787826740.981000:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826740.981004:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826740.981010:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826740.981023:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011510400/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826740.981033:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011510144:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787826740.981038:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826740.981047:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826740.981049:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826740.981060:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2144 00000100:00080000:0.0:1787826740.981064:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826740.981067:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826740.981072:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826740.981086:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908461c0 x1874670011510528/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787826740.981152:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826740.981205:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826740/real 1787826740] req@ffff907d908470c0 x1874670011510144/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826740.981237:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011510144/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826740.981249:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826740.981254:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826740.981258:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011510144:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826740.981267:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011510272:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826740.981292:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826740.981305:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826740/real 1787826740] req@ffff907d90847c00 x1874670011510272/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826740.981317:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011510272/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826740.981325:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826740.981329:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826740.981332:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011510272:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826740.981361:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011510400:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826740.981387:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826740.981399:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826740/real 1787826740] req@ffff907d90846580 x1874670011510400/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826740.981410:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011510400/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826740.981417:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826740.981420:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826740.981422:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011510400:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826740.981429:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011510528:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826740.981446:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826740.981457:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826740/real 1787826740] req@ffff907d908461c0 x1874670011510528/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826740.981469:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908461c0 x1874670011510528/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826740.981477:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826740.981480:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826740.981483:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011510528:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787826741.109257:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826742.132917:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826743.157009:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826744.180742:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826745.204869:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826746.100718:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826746.100727:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826746.100732:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826746.100741:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2149 00000100:00080000:0.0:1787826746.100751:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826746.100758:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826746.100792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847840 x1874670011510656/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826746.100826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826746.100834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826746.100836:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826746.100840:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2149 00000100:00080000:0.0:1787826746.100844:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826746.100848:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826746.100941:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011510656:192.168.203.104@tcp:250:kworker.0 00000008:00100000:0.0:1787826746.101003:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826746.101020:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908461c0 x1874670011510784/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826746.101029:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826746.101031:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826746.101033:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826746.101035:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2149 00000100:00080000:0.0:1787826746.101038:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787826746.101123:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826746.101158:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826746/real 1787826746] req@ffff907d90847840 x1874670011510656/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826746.101170:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847840 x1874670011510656/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787826746.101249:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826746.101253:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826746.101263:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011510912/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826746.101275:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826746.101278:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826746.101279:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826746.101281:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2149 00000100:00080000:0.0:1787826746.101284:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826746.101286:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826746.101289:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826746.101295:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011511040/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826746.102506:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826746.102512:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826746.102516:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011510656:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826746.102551:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011510784:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826746.102582:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826746.102596:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826746/real 1787826746] req@ffff907d908461c0 x1874670011510784/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826746.102629:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908461c0 x1874670011510784/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826746.102643:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826746.102646:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826746.102650:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011510784:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826746.102655:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011510912:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826746.102672:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826746.102681:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826746/real 1787826746] req@ffff907d90846580 x1874670011510912/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826746.102692:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011510912/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826746.102699:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826746.102702:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826746.102705:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011510912:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826746.102710:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011511040:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826746.102722:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826746.102730:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826746/real 1787826746] req@ffff907d90847c00 x1874670011511040/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826746.102739:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011511040/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826746.102747:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826746.102749:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826746.102753:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011511040:192.168.203.104@tcp:8:kworker.0 00010000:00010000:0.0:1787826746.228774:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826747.252951:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826748.279231:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826749.301011:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826750.324783:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826751.220752:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826751.220766:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826751.220772:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826751.220784:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2154 00000100:00080000:2.0:1787826751.220798:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826751.220806:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826751.220853:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011511168/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826751.220890:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826751.220894:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826751.220896:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826751.220901:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2154 00000100:00080000:2.0:1787826751.220904:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826751.220908:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826751.220921:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826751.220935:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011511296/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826751.220946:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787826751.220949:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011511168:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787826751.220950:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826751.220954:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826751.220960:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2154 00000100:00080000:2.0:1787826751.220964:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826751.220967:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826751.220974:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826751.220986:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011511424/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826751.220996:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826751.220999:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826751.221001:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826751.221004:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2154 00000100:00080000:2.0:1787826751.221010:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826751.221013:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826751.221018:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787826751.221024:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787826751.221030:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011511552/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826751.221074:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826751/real 1787826751] req@ffff907d91148f00 x1874670011511168/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826751.221089:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011511168/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826751.221097:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826751.221100:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826751.221103:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011511168:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826751.221125:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011511296:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826751.221141:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826751.221163:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826751/real 1787826751] req@ffff907d9114a940 x1874670011511296/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826751.221172:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011511296/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826751.221177:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826751.221179:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826751.221181:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011511296:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826751.221185:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011511424:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826751.221202:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826751.221209:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826751/real 1787826751] req@ffff907d911492c0 x1874670011511424/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826751.221215:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011511424/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826751.221220:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826751.221221:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826751.221223:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011511424:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826751.221226:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011511552:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826751.221243:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826751.221254:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826751/real 1787826751] req@ffff907d91148b40 x1874670011511552/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826751.221265:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011511552/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826751.221272:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826751.221275:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826751.221278:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011511552:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787826751.348798:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826752.372784:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826753.396856:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826754.420742:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826755.444854:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826756.340767:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826756.340781:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826756.340786:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826756.340798:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2160 00000100:00080000:2.0:1787826756.340812:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826756.340820:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826756.340871:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011511680/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826756.340909:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826756.340913:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826756.340916:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826756.340920:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2160 00000100:00080000:2.0:1787826756.340924:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826756.340929:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826756.340941:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826756.340953:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011511808/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826756.340963:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826756.340967:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826756.340968:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826756.340973:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2160 00000100:00080000:2.0:1787826756.340976:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826756.340981:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826756.340987:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826756.340996:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011511936/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826756.341006:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787826756.341011:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011511680:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787826756.341012:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826756.341014:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826756.341023:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2160 00000100:00080000:2.0:1787826756.341027:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826756.341030:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826756.341035:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826756.341047:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011512064/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787826756.341094:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826756.341156:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826756/real 1787826756] req@ffff907d91148b40 x1874670011511680/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826756.341180:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011511680/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826756.341190:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826756.341194:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826756.341201:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011511680:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826756.341208:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011511808:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826756.341229:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826756.341239:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826756/real 1787826756] req@ffff907d911492c0 x1874670011511808/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826756.341250:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011511808/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826756.341257:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826756.341260:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826756.341263:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011511808:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826756.341286:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011511936:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826756.341303:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826756.341311:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826756/real 1787826756] req@ffff907d9114a940 x1874670011511936/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826756.341321:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011511936/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826756.341334:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826756.341336:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826756.341339:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011511936:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826756.341344:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011512064:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826756.341357:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826756.341365:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826756/real 1787826756] req@ffff907d91148f00 x1874670011512064/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826756.341375:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011512064/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826756.341382:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826756.341386:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826756.341388:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011512064:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826756.468861:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826757.492795:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826758.516872:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826759.540774:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826760.564779:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826761.460807:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826761.460828:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826761.460832:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826761.460840:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2165 00000100:00080000:1.0:1787826761.460851:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826761.460858:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826761.460893:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011512192/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826761.460925:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826761.460929:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826761.460931:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826761.460935:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2165 00000100:00080000:1.0:1787826761.460939:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826761.460943:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826761.460953:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826761.460966:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011512320/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826761.460973:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011512192:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787826761.460979:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826761.460984:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826761.460985:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826761.460990:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2165 00000100:00080000:1.0:1787826761.460993:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826761.460998:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826761.461002:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826761.461014:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011512448/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826761.461023:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826761.461027:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826761.461029:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826761.461032:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2165 00000100:00080000:1.0:1787826761.461035:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826761.461040:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:3.0:1787826761.461042:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:1.0:1787826761.461046:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826761.461059:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011512576/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826761.461100:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826761/real 1787826761] req@ffff907d908a0000 x1874670011512192/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826761.461120:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011512192/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826761.461129:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826761.461134:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826761.461140:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011512192:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826761.461165:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011512320:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826761.461196:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826761.461213:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826761/real 1787826761] req@ffff907d908a2d00 x1874670011512320/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826761.461225:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011512320/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826761.461234:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826761.461238:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826761.461243:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011512320:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826761.461250:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011512448:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826761.461273:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826761.461291:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826761/real 1787826761] req@ffff907d908a1e00 x1874670011512448/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826761.461304:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011512448/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826761.461313:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826761.461317:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826761.461321:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011512448:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826761.461330:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011512576:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826761.461352:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826761.461460:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826761/real 1787826761] req@ffff907d908a2940 x1874670011512576/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826761.461474:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011512576/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826761.461484:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826761.461488:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826761.461492:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011512576:192.168.203.104@tcp:8:kworker.0 00010000:00010000:0.0:1787826761.588829:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826762.612801:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826763.636803:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826764.660765:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826765.684823:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826766.580741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826766.580752:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826766.580756:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826766.580766:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2170 00000100:00080000:0.0:1787826766.580776:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826766.580793:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826766.580830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011512704/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826766.580869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826766.580873:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826766.580874:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826766.580878:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2170 00000100:00080000:0.0:1787826766.580882:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826766.580886:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826766.580895:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826766.580915:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011512832/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826766.580921:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826766.580924:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826766.580925:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826766.580928:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2170 00000100:00080000:0.0:1787826766.580930:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826766.580933:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826766.580936:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826766.580940:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011512960/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826766.580946:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826766.580948:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826766.580949:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826766.580950:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2170 00000100:00080000:0.0:1787826766.580952:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826766.580953:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826766.580956:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826766.580960:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011513088/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826766.581034:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011512704:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787826766.581107:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826766.581143:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826766/real 1787826766] req@ffff907d908470c0 x1874670011512704/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826766.581159:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011512704/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826766.581169:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826766.581173:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826766.581177:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011512704:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826766.581183:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011512832:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826766.581206:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826766.581217:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826766/real 1787826766] req@ffff907d90845e00 x1874670011512832/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826766.581225:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011512832/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826766.581233:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826766.581236:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826766.581239:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011512832:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826766.581244:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011512960:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826766.581263:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826766.581274:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826766/real 1787826766] req@ffff907d90844b40 x1874670011512960/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826766.581285:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011512960/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826766.581290:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826766.581292:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826766.581294:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011512960:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826766.581298:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011513088:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826766.581311:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826766.581317:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826766/real 1787826766] req@ffff907d908443c0 x1874670011513088/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826766.581323:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011513088/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826766.581328:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826766.581329:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826766.581332:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011513088:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826766.708762:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826767.732797:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826768.756795:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826769.780718:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826770.804741:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826771.700722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826771.700732:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826771.700736:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826771.700745:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2175 00000100:00080000:0.0:1787826771.700755:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826771.700762:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826771.700807:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011513216/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826771.700834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826771.700838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826771.700840:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826771.700844:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2175 00000100:00080000:0.0:1787826771.700847:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826771.700851:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826771.700861:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826771.700873:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011513344/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826771.700879:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011513216:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787826771.700890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826771.700894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826771.700896:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826771.700900:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2175 00000100:00080000:0.0:1787826771.700904:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826771.700908:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826771.700913:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826771.700924:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011513472/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826771.700934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826771.700938:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826771.700940:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826771.700944:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2175 00000100:00080000:0.0:1787826771.700948:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787826771.700953:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787826771.700956:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826771.700963:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826771.700976:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011513600/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826771.700992:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826771/real 1787826771] req@ffff907d90844000 x1874670011513216/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826771.701005:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011513216/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826771.701026:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826771.701029:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826771.701032:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011513216:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826771.701048:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011513344:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826771.701072:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826771.701085:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826771/real 1787826771] req@ffff907d90844780 x1874670011513344/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826771.701097:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011513344/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826771.701117:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826771.701120:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826771.701123:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011513344:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826771.701140:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011513472:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826771.701160:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826771.701169:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826771/real 1787826771] req@ffff907d90845680 x1874670011513472/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826771.701176:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011513472/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826771.701181:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826771.701183:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826771.701185:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011513472:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826771.701188:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011513600:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826771.701200:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826771.701209:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826771/real 1787826771] req@ffff907d90846d00 x1874670011513600/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826771.701219:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011513600/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826771.701227:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826771.701230:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826771.701232:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011513600:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826771.828766:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826772.852763:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826773.876767:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826774.900769:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826775.924852:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826776.820712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826776.820725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826776.820729:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826776.820746:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2180 00000100:00080000:0.0:1787826776.820761:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826776.820769:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826776.820817:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011513728/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826776.820895:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011513728:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787826776.820957:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787826776.820998:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826776.821002:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826776.821004:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826776.821008:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2180 00000100:00080000:0.0:1787826776.821013:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826776.821017:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826776.821027:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826776.821039:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011513856/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826776.821105:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826776/real 1787826776] req@ffff907d908452c0 x1874670011513728/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826776.821129:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011513728/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826776.821144:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826776.821148:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826776.821152:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011513728:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826776.821176:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011513856:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826776.821204:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826776.821217:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826776/real 1787826776] req@ffff907d90845a40 x1874670011513856/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826776.821228:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011513856/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826776.821235:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826776.821238:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826776.821241:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011513856:192.168.203.104@tcp:38:kworker.0 00000100:00080000:0.0:1787826776.821352:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826776.821357:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826776.821359:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826776.821363:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2180 00000100:00080000:0.0:1787826776.821367:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826776.821372:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826776.821377:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826776.821388:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011513984/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826776.821405:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826776.821413:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826776.821414:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826776.821418:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2180 00000100:00080000:0.0:1787826776.821422:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826776.821426:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826776.821431:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826776.821440:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011513984:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787826776.821444:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011514112/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787826776.821473:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826776.821485:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826776/real 1787826776] req@ffff907d90844f00 x1874670011513984/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826776.821494:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011513984/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826776.821506:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826776.821509:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826776.821512:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011513984:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826776.821525:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011514112:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826776.821543:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826776.821554:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826776/real 1787826776] req@ffff907d90846940 x1874670011514112/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826776.821565:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011514112/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826776.821572:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826776.821574:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826776.821576:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011514112:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787826776.948784:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826777.972782:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826778.996839:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826780.020874:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826781.044881:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787826781.941201:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826781.941216:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787826781.941221:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826781.941234:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2185 00000100:00080000:3.0:1787826781.941247:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826781.941256:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826781.941305:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011514240/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826781.941344:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826781.941349:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787826781.941351:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826781.941355:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2185 00000100:00080000:3.0:1787826781.941359:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826781.941363:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826781.941375:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826781.941389:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011514368/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826781.941399:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826781.941402:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787826781.941404:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826781.941407:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2185 00000100:00080000:3.0:1787826781.941411:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826781.941415:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826781.941420:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826781.941431:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011514496/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826781.941440:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826781.941445:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787826781.941447:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826781.941451:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2185 00000100:00080000:3.0:1787826781.941461:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826781.941466:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826781.941471:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826781.941482:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011514624/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826781.941499:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011514240:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826781.941655:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011514368:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826781.941689:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011514496:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826781.941725:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011514624:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787826782.069763:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826783.092876:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826784.117980:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787826785.012901:0:2337:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:0.0:1787826785.012956:0:2337:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787826785.012972:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826785.013011:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826785.013019:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826785.013025:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:1.0:1787826785.013063:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826781/real 1787826785] req@ffff907da562da40 x1874670011514624/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826785.013086:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011514624/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826785.013097:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826785.013120:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826785.013125:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011514624:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826785.013134:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826781/real 1787826785] req@ffff907da562cb40 x1874670011514240/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826785.013147:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011514240/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826785.013154:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826785.013158:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826785.013161:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011514240:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826785.013168:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826781/real 1787826785] req@ffff907da562de00 x1874670011514368/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826785.013179:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011514368/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826785.013187:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826785.013190:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826785.013193:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011514368:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826785.013200:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826781/real 1787826785] req@ffff907da562f840 x1874670011514496/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826785.013211:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011514496/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826785.013218:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826785.013226:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826785.013230:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011514496:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787826785.140805:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826786.164787:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826787.060777:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826787.060791:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826787.060796:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826787.060809:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2190 00000100:00080000:2.0:1787826787.060823:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826787.060831:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826787.060880:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011514752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826787.060991:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011514752:192.168.203.104@tcp:250:kworker.0 00000800:00000100:1.0:1787826787.061066:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787826787.061099:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826787.061103:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826787.061105:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826787.061109:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2190 00000100:00080000:2.0:1787826787.061113:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826787.061117:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826787.061128:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826787.061142:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011514880/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826787.061208:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826787/real 1787826787] req@ffff907d9114ad00 x1874670011514752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826787.061225:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011514752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826787.061236:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826787.061239:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826787.061243:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011514752:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826787.061272:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011514880:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826787.061297:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826787.061311:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826787/real 1787826787] req@ffff907d91149a40 x1874670011514880/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826787.061323:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011514880/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826787.061331:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826787.061334:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826787.061337:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011514880:192.168.203.104@tcp:38:kworker.0 00000100:00080000:2.0:1787826787.061439:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826787.061444:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826787.061446:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826787.061450:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2190 00000100:00080000:2.0:1787826787.061455:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826787.061459:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826787.061466:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826787.061478:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011515008/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826787.061520:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011515008:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826787.061544:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787826787.061566:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826787.061570:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826787.061572:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826787.061576:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2190 00000100:00080000:2.0:1787826787.061581:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826787.061585:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826787.061591:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826787.061604:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011515136/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826787.061699:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826787/real 1787826787] req@ffff907d9114a580 x1874670011515008/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826787.061711:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011515008/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826787.061719:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826787.061723:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826787.061726:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011515008:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826787.061738:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011515136:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826787.061758:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826787.061769:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826787/real 1787826787] req@ffff907d9114b840 x1874670011515136/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826787.061779:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011515136/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826787.061792:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826787.061795:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826787.061799:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011515136:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826787.188959:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826788.212838:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826789.236957:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826790.260811:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826791.284912:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826792.180722:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826792.180732:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826792.180737:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826792.180746:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2195 00000100:00080000:2.0:1787826792.180756:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826792.180768:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826792.180811:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011515264/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826792.180841:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826792.180845:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826792.180846:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826792.180851:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2195 00000100:00080000:2.0:1787826792.180855:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826792.180859:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826792.180878:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826792.180889:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011515392/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826792.180898:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787826792.180903:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011515264:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787826792.180903:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826792.180906:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826792.180912:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2195 00000100:00080000:2.0:1787826792.180915:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826792.180918:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826792.180921:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826792.180929:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011515520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826792.180935:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826792.180937:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826792.180938:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826792.180940:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2195 00000100:00080000:2.0:1787826792.180942:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826792.180944:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826792.180946:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826792.180952:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011515648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787826792.180978:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826792.181017:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826792/real 1787826792] req@ffff907d91149680 x1874670011515264/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826792.181034:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011515264/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826792.181045:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826792.181049:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826792.181053:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011515264:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826792.181074:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011515392:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826792.181096:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826792.181108:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826792/real 1787826792] req@ffff907d9114bc00 x1874670011515392/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826792.181119:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011515392/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826792.181146:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826792.181150:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826792.181153:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011515392:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826792.181159:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011515520:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826792.181180:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826792.181191:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826792/real 1787826792] req@ffff907d911483c0 x1874670011515520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826792.181203:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011515520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826792.181210:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826792.181213:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826792.181217:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011515520:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826792.181223:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011515648:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826792.181240:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826792.181258:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826792/real 1787826792] req@ffff907d9114b480 x1874670011515648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826792.181271:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011515648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826792.181279:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826792.181282:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826792.181285:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011515648:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787826792.308791:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826793.332874:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826794.356768:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826795.380785:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826796.404773:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826797.300789:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826797.300809:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826797.300814:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826797.300824:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2201 00000100:00080000:2.0:1787826797.300844:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826797.300851:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826797.300884:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011515776/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826797.300913:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826797.300917:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826797.300920:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826797.300924:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2201 00000100:00080000:2.0:1787826797.300929:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826797.300934:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826797.300944:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826797.300958:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011515904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826797.300967:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826797.300969:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826797.300970:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826797.300974:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2201 00000100:00080000:2.0:1787826797.300976:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826797.300979:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826797.300982:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826797.300988:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011516032/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826797.300993:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826797.300995:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826797.300996:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826797.300998:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2201 00000100:00080000:2.0:1787826797.301000:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826797.301002:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826797.301004:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826797.301009:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011516160/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826797.301122:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011515776:192.168.203.104@tcp:250:kworker.0 00000800:00000100:1.0:1787826797.301176:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826797.301208:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826797/real 1787826797] req@ffff907d9114b480 x1874670011515776/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826797.301221:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011515776/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826797.301229:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826797.301232:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826797.301243:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011515776:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826797.301250:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011515904:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826797.301272:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826797.301284:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826797/real 1787826797] req@ffff907d911483c0 x1874670011515904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826797.301302:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011515904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826797.301307:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826797.301309:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826797.301311:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011515904:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826797.301315:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011516032:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826797.301332:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826797.301343:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826797/real 1787826797] req@ffff907d9114bc00 x1874670011516032/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826797.301354:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011516032/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826797.301362:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826797.301366:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826797.301369:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011516032:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826797.301374:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011516160:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826797.301395:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826797.301406:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826797/real 1787826797] req@ffff907d91149680 x1874670011516160/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826797.301418:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011516160/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826797.301425:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826797.301428:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826797.301431:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011516160:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826797.428804:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826798.452786:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826799.476782:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826800.500846:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826801.524740:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826802.420702:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826802.420709:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826802.420712:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826802.420719:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2206 00000100:00080000:2.0:1787826802.420729:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826802.420734:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826802.420765:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011516288/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826802.420800:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826802.420804:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826802.420806:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826802.420810:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2206 00000100:00080000:2.0:1787826802.420814:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826802.420819:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826802.420827:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826802.420839:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011516288:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787826802.420840:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011516416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826802.420853:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826802.420855:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826802.420857:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826802.420860:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2206 00000100:00080000:2.0:1787826802.420862:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826802.420864:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826802.420867:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826802.420877:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011516544/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826802.420886:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826802.420889:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826802.420891:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826802.420894:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2206 00000100:00080000:2.0:1787826802.420898:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826802.420902:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:1.0:1787826802.420906:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:2.0:1787826802.420910:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826802.420936:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011516672/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826802.420947:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826802/real 1787826802] req@ffff907d9114b840 x1874670011516288/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826802.420971:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011516288/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826802.420981:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826802.420984:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826802.420988:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011516288:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826802.421010:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011516416:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826802.421032:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826802.421045:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826802/real 1787826802] req@ffff907d9114a580 x1874670011516416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826802.421056:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011516416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826802.421063:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826802.421065:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826802.421068:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011516416:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826802.421073:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011516544:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826802.421093:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826802.421104:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826802/real 1787826802] req@ffff907d91149a40 x1874670011516544/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826802.421114:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011516544/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826802.421127:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826802.421130:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826802.421132:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011516544:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826802.421138:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011516672:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826802.421156:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826802.421167:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826802/real 1787826802] req@ffff907d9114ad00 x1874670011516672/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826802.421177:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011516672/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826802.421184:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826802.421187:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826802.421190:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011516672:192.168.203.104@tcp:8:kworker.0 00010000:00010000:0.0:1787826802.548734:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826803.572773:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826804.596787:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826805.620787:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826806.645300:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787826807.540794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826807.540806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787826807.540812:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826807.540825:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2211 00000100:00080000:3.0:1787826807.540840:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826807.540848:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826807.540899:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011516800/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826807.540935:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826807.540939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787826807.540941:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826807.540945:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2211 00000100:00080000:3.0:1787826807.540948:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826807.540953:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826807.540971:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826807.540983:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011516800:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826807.540991:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011516928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826807.541004:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826807.541008:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787826807.541010:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826807.541014:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2211 00000100:00080000:3.0:1787826807.541019:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826807.541024:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826807.541029:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826807.541041:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011517056/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826807.541050:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826807.541054:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1787826807.541057:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787826807.541058:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826807.541064:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2211 00000100:00080000:3.0:1787826807.541071:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826807.541077:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826807.541082:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826807.541108:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011517184/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826807.541128:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826807/real 1787826807] req@ffff907da562e1c0 x1874670011516800/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826807.541160:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011516800/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826807.541178:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826807.541183:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826807.541187:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011516800:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826807.541213:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011516928:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826807.541245:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826807.541263:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826807/real 1787826807] req@ffff907da562e940 x1874670011516928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826807.541275:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011516928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826807.541283:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826807.541286:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826807.541289:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011516928:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826807.541296:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011517056:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826807.541324:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826807.541336:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826807/real 1787826807] req@ffff907da562fc00 x1874670011517056/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826807.541345:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011517056/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826807.541352:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826807.541355:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826807.541358:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011517056:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826807.541365:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011517184:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826807.541382:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826807.541398:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826807/real 1787826807] req@ffff907da562f0c0 x1874670011517184/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826807.541409:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011517184/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826807.541416:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826807.541419:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826807.541421:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011517184:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826807.668792:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826808.692868:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826809.717430:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826810.741136:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826811.764845:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787826812.660733:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826812.660742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787826812.660746:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826812.660754:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2216 00000100:00080000:3.0:1787826812.660764:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826812.660769:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826812.660800:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011517312/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826812.660827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826812.660831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787826812.660832:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826812.660836:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2216 00000100:00080000:3.0:1787826812.660840:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826812.660843:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826812.660853:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826812.660864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011517440/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826812.660874:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826812.660877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787826812.660878:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826812.660881:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2216 00000100:00080000:3.0:1787826812.660885:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826812.660888:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826812.660892:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826812.660902:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011517568/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826812.660916:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826812.660919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787826812.660921:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826812.660924:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2216 00000100:00080000:3.0:1787826812.660928:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826812.660931:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826812.660935:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826812.660944:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011517696/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826812.660955:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011517312:192.168.203.104@tcp:250:kworker.0 00000800:00000100:1.0:1787826812.661008:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826812.661048:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826812/real 1787826812] req@ffff907da562c3c0 x1874670011517312/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826812.661070:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011517312/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826812.661080:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826812.661084:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826812.661088:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011517312:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826812.661095:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011517440:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826812.661120:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826812.661146:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826812/real 1787826812] req@ffff907da562d2c0 x1874670011517440/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826812.661159:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011517440/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826812.661167:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826812.661170:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826812.661173:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011517440:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826812.661179:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011517568:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826812.661196:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826812.661203:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826812/real 1787826812] req@ffff907da562e580 x1874670011517568/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826812.661210:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011517568/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826812.661215:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826812.661217:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826812.661219:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011517568:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826812.661245:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011517696:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826812.661264:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826812.661275:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826812/real 1787826812] req@ffff907da562c000 x1874670011517696/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826812.661292:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011517696/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826812.661298:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826812.661301:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826812.661304:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011517696:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826812.788788:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826813.812781:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826814.836791:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826815.860857:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826816.884755:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826817.780730:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826817.780739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826817.780744:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826817.780754:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2221 00000100:00080000:0.0:1787826817.780766:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826817.780775:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826817.780823:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011517824/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826817.780854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826817.780858:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826817.780860:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826817.780863:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2221 00000100:00080000:0.0:1787826817.780867:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826817.780871:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826817.780883:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826817.780908:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011517952/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826817.780912:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011517824:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787826817.780925:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826817.780930:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826817.780931:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826817.780937:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2221 00000100:00080000:0.0:1787826817.780941:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826817.780945:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826817.780950:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826817.780962:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011518080/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826817.780971:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826817.780975:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826817.780977:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826817.780981:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2221 00000100:00080000:0.0:1787826817.780984:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826817.780990:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:1.0:1787826817.780992:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:0.0:1787826817.780996:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826817.781008:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011518208/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826817.781037:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826817/real 1787826817] req@ffff907d90847480 x1874670011517824/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826817.781052:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011517824/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826817.781060:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826817.781063:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826817.781066:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011517824:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826817.781087:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011517952:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826817.781111:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826817.781119:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826817/real 1787826817] req@ffff907d90846940 x1874670011517952/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826817.781131:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011517952/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826817.781136:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826817.781138:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826817.781140:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011517952:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826817.781143:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011518080:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826817.781156:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826817.781165:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826817/real 1787826817] req@ffff907d90844f00 x1874670011518080/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826817.781177:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011518080/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826817.781185:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826817.781188:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826817.781191:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011518080:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826817.781197:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011518208:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826817.781214:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826817.781222:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826817/real 1787826817] req@ffff907d90845a40 x1874670011518208/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826817.781229:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011518208/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826817.781238:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826817.781240:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826817.781242:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011518208:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787826817.908849:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826818.932832:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826819.956937:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826820.980783:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826822.005276:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826822.901523:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826822.901537:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826822.901543:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826822.901555:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2226 00000100:00080000:2.0:1787826822.901568:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826822.901576:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826822.901677:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011518336/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826822.901711:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826822.901715:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826822.901717:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826822.901721:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2226 00000100:00080000:2.0:1787826822.901725:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826822.901728:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826822.901741:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826822.901755:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011518464/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826822.901765:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787826822.901769:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011518336:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787826822.901770:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826822.901773:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826822.901779:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2226 00000100:00080000:2.0:1787826822.901782:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826822.901785:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826822.901790:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826822.901801:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011518592/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826822.901816:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826822.901820:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826822.901821:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826822.901825:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2226 00000100:00080000:2.0:1787826822.901828:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826822.901832:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826822.901836:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826822.901846:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011518720/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787826822.901850:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826822.901909:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826822/real 1787826822] req@ffff907d91148f00 x1874670011518336/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826822.901929:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011518336/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826822.901940:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826822.901944:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826822.901948:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011518336:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826822.901974:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011518464:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826822.901993:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826822.902002:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826822/real 1787826822] req@ffff907d9114a940 x1874670011518464/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826822.902014:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011518464/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826822.902027:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826822.902031:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826822.902034:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011518464:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826822.902041:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011518592:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826822.902061:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826822.902072:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826822/real 1787826822] req@ffff907d911492c0 x1874670011518592/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826822.902084:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011518592/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826822.902092:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826822.902096:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826822.902098:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011518592:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826822.902102:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011518720:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826822.902129:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826822.902140:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826822/real 1787826822] req@ffff907d91148b40 x1874670011518720/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826822.902151:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011518720/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826822.902159:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826822.902162:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826822.902164:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011518720:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787826823.028762:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826824.052868:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826825.076770:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826826.101696:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826827.124794:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826828.020735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826828.020742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826828.020751:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826828.020761:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2231 00000100:00080000:1.0:1787826828.020772:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826828.020788:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826828.020830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011518848/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826828.020860:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826828.020863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826828.020865:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826828.020868:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2231 00000100:00080000:1.0:1787826828.020872:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826828.020876:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826828.020885:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826828.020897:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011518976/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826828.020905:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826828.020908:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826828.020909:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826828.020912:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2231 00000100:00080000:1.0:1787826828.020915:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826828.020919:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826828.020923:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826828.020931:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011519104/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826828.020939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826828.020942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826828.020944:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826828.020947:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2231 00000100:00080000:1.0:1787826828.020949:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826828.020952:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826828.020956:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826828.020964:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011519232/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826828.021037:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011518848:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787826828.021089:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826828.021125:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826828/real 1787826828] req@ffff907d908a3840 x1874670011518848/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826828.021141:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011518848/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826828.021151:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826828.021154:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826828.021157:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011518848:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826828.021163:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011518976:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826828.021183:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826828.021193:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826828/real 1787826828] req@ffff907d908a2580 x1874670011518976/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826828.021204:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011518976/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826828.021212:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826828.021215:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826828.021218:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011518976:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826828.021223:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011519104:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826828.021243:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826828.021252:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826828/real 1787826828] req@ffff907d908a21c0 x1874670011519104/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826828.021260:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011519104/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826828.021265:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826828.021267:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826828.021269:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011519104:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826828.021273:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011519232:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826828.021285:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826828.021291:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826828/real 1787826828] req@ffff907d908a12c0 x1874670011519232/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826828.021357:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011519232/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826828.021364:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826828.021366:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826828.021368:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011519232:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826828.149402:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826829.173741:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826830.196816:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826831.221077:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826832.244857:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826833.141192:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826833.141201:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826833.141206:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826833.141218:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2236 00000100:00080000:0.0:1787826833.141229:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826833.141237:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826833.141273:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011519360/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826833.141315:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826833.141320:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826833.141322:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826833.141328:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2236 00000100:00080000:0.0:1787826833.141333:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826833.141337:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826833.141348:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826833.141366:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011519488/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826833.141381:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826833.141385:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826833.141389:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826833.141393:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2236 00000100:00080000:0.0:1787826833.141399:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826833.141402:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826833.141410:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826833.141412:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011519360:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787826833.141425:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011519616/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826833.141432:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826833.141435:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826833.141437:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826833.141439:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2236 00000100:00080000:0.0:1787826833.141442:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826833.141445:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826833.141448:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826833.141454:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011519744/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787826833.141507:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826833.141573:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826833/real 1787826833] req@ffff907d908452c0 x1874670011519360/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826833.141631:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011519360/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826833.141644:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826833.141650:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826833.141656:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011519360:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826833.141768:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011519488:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826833.141954:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826833.141972:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826833/real 1787826833] req@ffff907d90846d00 x1874670011519488/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826833.141987:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011519488/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826833.142044:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826833.142049:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826833.142054:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011519488:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826833.142064:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011519616:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826833.142090:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826833.142103:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826833/real 1787826833] req@ffff907d90845680 x1874670011519616/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826833.142115:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011519616/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826833.142123:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826833.142127:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826833.142130:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011519616:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826833.142137:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011519744:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826833.142157:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826833.142168:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826833/real 1787826833] req@ffff907d90844780 x1874670011519744/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826833.142180:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011519744/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826833.142188:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826833.142191:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826833.142195:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011519744:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787826833.268894:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826834.292846:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826835.316760:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826836.340788:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826837.364762:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826838.260741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826838.260753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826838.260758:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826838.260770:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2242 00000100:00080000:0.0:1787826838.260784:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826838.260792:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826838.260836:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011519872/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826838.260867:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826838.260870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826838.260872:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826838.260875:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2242 00000100:00080000:0.0:1787826838.260878:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826838.260882:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826838.260893:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826838.260905:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011520000/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826838.260914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826838.260918:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826838.260920:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826838.260923:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2242 00000100:00080000:0.0:1787826838.260926:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826838.260931:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826838.260933:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011519872:192.168.203.104@tcp:250:kworker.0 00000008:00100000:0.0:1787826838.260936:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826838.260950:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011520128/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826838.260959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826838.260962:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826838.260964:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826838.260967:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2242 00000100:00080000:0.0:1787826838.260970:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826838.260974:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826838.260979:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826838.260989:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011520256/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787826838.261010:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826838.261057:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826838/real 1787826838] req@ffff907d90844000 x1874670011519872/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826838.261076:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011519872/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826838.261086:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826838.261091:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826838.261095:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011519872:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826838.261103:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011520000:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826838.261127:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826838.261139:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826838/real 1787826838] req@ffff907d908443c0 x1874670011520000/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826838.261150:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011520000/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826838.261158:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826838.261161:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826838.261164:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011520000:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826838.261192:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011520128:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826838.261214:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826838.261222:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826838/real 1787826838] req@ffff907d90844b40 x1874670011520128/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826838.261229:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011520128/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826838.261234:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826838.261236:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826838.261239:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011520128:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826838.261242:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011520256:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826838.261252:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826838.261257:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826838/real 1787826838] req@ffff907d90845e00 x1874670011520256/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826838.261264:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011520256/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826838.261271:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826838.261274:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826838.261277:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011520256:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787826838.388873:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826839.412846:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826840.436790:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826841.460816:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826842.484767:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826843.380711:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826843.380720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826843.380724:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826843.380733:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2247 00000100:00080000:0.0:1787826843.380743:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826843.380751:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826843.380785:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011520384/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826843.380809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826843.380813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826843.380815:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826843.380818:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2247 00000100:00080000:0.0:1787826843.380821:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826843.380825:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826843.380834:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826843.380935:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011520384:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787826843.381019:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826843.381073:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826843/real 1787826843] req@ffff907d908470c0 x1874670011520384/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826843.381093:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011520384/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826843.381106:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826843.381110:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826843.381114:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011520384:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787826843.381577:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011520512/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826843.381604:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826843.381631:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826843.381633:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826843.381637:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2247 00000100:00080000:0.0:1787826843.381642:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826843.381647:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826843.381653:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826843.381680:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011520640/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826843.381689:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826843.381692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826843.381694:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826843.381697:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2247 00000100:00080000:0.0:1787826843.381701:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826843.381705:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826843.381709:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826843.381718:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908461c0 x1874670011520768/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826843.381818:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011520512:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826843.381884:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826843.381908:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826843/real 1787826843] req@ffff907d90847c00 x1874670011520512/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826843.381924:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011520512/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826843.381934:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826843.381937:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826843.381941:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011520512:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826843.381948:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011520640:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826843.381968:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826843.381978:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826843/real 1787826843] req@ffff907d90846580 x1874670011520640/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826843.381989:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011520640/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826843.382002:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826843.382005:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826843.382008:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011520640:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826843.382014:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011520768:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826843.382031:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826843.382041:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826843/real 1787826843] req@ffff907d908461c0 x1874670011520768/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826843.382051:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908461c0 x1874670011520768/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826843.382058:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826843.382061:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826843.382064:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908461c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011520768:192.168.203.104@tcp:8:kworker.0 00010000:00010000:0.0:1787826843.508955:0:61297:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826844.532945:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826845.556851:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826846.581004:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826847.604752:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787826848.500754:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826848.500763:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787826848.500768:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826848.500778:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2252 00000100:00080000:3.0:1787826848.500790:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826848.500797:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826848.500835:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011520896/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826848.500867:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826848.500871:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787826848.500873:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826848.500877:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2252 00000100:00080000:3.0:1787826848.500889:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826848.500893:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826848.500902:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826848.500914:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011521024/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826848.500923:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826848.500927:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787826848.500929:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826848.500932:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2252 00000100:00080000:3.0:1787826848.500935:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826848.500939:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826848.500943:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826848.500952:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011521152/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826848.500960:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826848.500964:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787826848.500965:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826848.500968:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2252 00000100:00080000:3.0:1787826848.500971:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826848.500974:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826848.500979:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826848.500987:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011521280/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826848.501067:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011520896:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826848.501157:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011521024:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826848.501184:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011521152:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826848.501206:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011521280:192.168.203.104@tcp:8:kworker.0 00010000:00010000:0.0:1787826848.628784:0:64869:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 3/3 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1787826848.628841:0:60472:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 3 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1787826848.628852:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b619 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x28400020000 nid: local remote: 0x7918213f4fc0f54a expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826848.628881:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1dd66eb/0xe5f736940357b60b lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400020000 nid: local remote: 0x7918213f4fc0f535 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1787826848.628889:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1787826848.628905:0:60472:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fb:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1787826848.628920:0:60472:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fb:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1787826848.628934:0:60472:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000066987f3b/0xe5f736940357b524 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x28400020000 nid: local remote: 0x7918213f4fc0f241 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826848.628974:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b619 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4829400020020 nid: local remote: 0x7918213f4fc0f54a expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826848.628983:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1dd66eb/0xe5f736940357b60b lrc: 1/0,0 mode: --/PW res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400020020 nid: local remote: 0x7918213f4fc0f535 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826848.628990:0:60472:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000066987f3b/0xe5f736940357b524 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4829400020020 nid: local remote: 0x7918213f4fc0f241 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826848.628996:0:60472:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:1.0:1787826848.629021:0:60472:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 000000007e8b8664/0xe5f736940357b619 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x321:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4829400020020 nid: local remote: 0x7918213f4fc0f54a expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1787826848.629048:0:60472:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 00000000e1dd66eb/0xe5f736940357b60b lrc: 0/0,0 mode: --/PW res: [0x240000400:0x322:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400020020 nid: local remote: 0x7918213f4fc0f535 expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000100:00080000:0.0:1787826848.629059:0:2348:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff907d908a30c0 x1874670011521408/t0(0) o103->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:17/18 lens 336/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1787826848.629064:0:60472:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff907d986d0000 lock: 0000000066987f3b/0xe5f736940357b524 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x31f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4829400020020 nid: local remote: 0x7918213f4fc0f241 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1787826848.629090:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826848.629092:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826848.629094:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826849.652774:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826850.678172:0:64869:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 12/12 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1787826850.678213:0:61297:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 12 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826850.678222:0:61297:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff907d986d0000 lock: ffff907d83f0a000/0xe5f736940357bae2 lrc: 2/0,0 mode: CR/CR res: [0x200000404:0x208:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7918213f4fc107cd expref: -99 pid: 69437 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 0 00010000:00010000:2.0:1787826850.678236:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000002:00100000:0.0:1787826850.678298:0:61297:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff907d908a03c0 x1874670011469056/t8589943033(8589943033) o101->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 2224/608 e 0 to 0 dl 1787826425 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1787826850.678309:0:61297:0:(genops.c:2423:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff907d986d0000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1787826850.678315:0:61297:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff907d90847840 x1874670011521664/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00080000:0.0:1787826850.678334:0:61297:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff907d90847840 x1874670011521664/t0(0) o35->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:23/10 lens 392/2200 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/600/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1787826850.678343:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000400:00000100:2.0:1787826851.572895:0:2338:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:2.0:1787826851.572941:0:2338:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787826851.572957:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787826851.572995:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787826851.573003:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787826851.573011:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:1.0:1787826851.573080:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826848/real 1787826851] req@ffff907da562f480 x1874670011520896/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826851.573103:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011520896/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826851.573114:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826851.573118:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826851.573122:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011520896:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826851.573130:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826848/real 1787826851] req@ffff907da562c780 x1874670011521024/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826851.573142:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011521024/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826851.573151:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826851.573154:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826851.573158:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011521024:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826851.573169:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826848/real 1787826851] req@ffff907da562ed00 x1874670011521152/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826851.573181:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011521152/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826851.573188:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826851.573192:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826851.573195:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011521152:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826851.573201:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826848/real 1787826851] req@ffff907da562cf00 x1874670011521280/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826851.573211:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011521280/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826851.573219:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826851.573222:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826851.573225:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011521280:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787826851.700718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787826851.700842:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826852.724747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787826852.724801:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826853.620845:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826853.620859:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826853.620863:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826853.620876:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2257 00000100:00080000:2.0:1787826853.620889:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826853.620897:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826853.620940:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011521792/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826853.620979:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826853.620983:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826853.620985:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826853.620990:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2257 00000100:00080000:2.0:1787826853.620994:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826853.620998:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826853.621010:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826853.621025:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011521920/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826853.621037:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826853.621041:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826853.621043:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826853.621047:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2257 00000100:00080000:2.0:1787826853.621051:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826853.621056:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826853.621062:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826853.621076:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011521792:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787826853.621077:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011522048/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826853.621103:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826853.621107:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826853.621109:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826853.621115:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2257 00000100:00080000:2.0:1787826853.621119:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826853.621123:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826853.621130:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826853.621143:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011522176/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787826853.621170:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826853.621220:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826853/real 1787826853] req@ffff907d91148b40 x1874670011521792/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826853.621237:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011521792/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826853.621247:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826853.621251:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826853.621255:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011521792:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826853.621263:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011521920:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826853.621286:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826853.621297:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826853/real 1787826853] req@ffff907d911492c0 x1874670011521920/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826853.621307:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011521920/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826853.621335:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826853.621339:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826853.621342:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011521920:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826853.621365:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011522048:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826853.621384:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826853.621394:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826853/real 1787826853] req@ffff907d9114a940 x1874670011522048/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826853.621404:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011522048/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826853.621412:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826853.621415:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826853.621418:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011522048:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826853.621424:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011522176:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826853.621454:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826853.621464:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826853/real 1787826853] req@ffff907d91148f00 x1874670011522176/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826853.621475:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011522176/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826853.621487:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826853.621490:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826853.621493:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011522176:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826853.749387:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826853.749399:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:0.0:1787826854.772669:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787826854.772734:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826855.796791:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826855.796839:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826856.820722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826856.820776:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826857.844712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826857.844795:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787826858.740742:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826858.740752:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787826858.740756:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826858.740766:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2262 00000100:00080000:3.0:1787826858.740778:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826858.740784:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826858.740823:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011522304/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826858.740846:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826858.740849:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787826858.740850:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826858.740853:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2262 00000100:00080000:3.0:1787826858.740855:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826858.740857:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826858.740866:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826858.740878:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011522432/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826858.740885:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826858.740887:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787826858.740889:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826858.740893:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2262 00000100:00080000:3.0:1787826858.740896:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826858.740899:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826858.740902:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826858.740908:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011522560/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826858.740915:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826858.740917:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787826858.740918:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826858.740921:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2262 00000100:00080000:3.0:1787826858.740923:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826858.740928:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826858.740932:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826858.740934:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011522304:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826858.740940:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011522688/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787826858.741026:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826858.741068:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826858/real 1787826858] req@ffff907da562d680 x1874670011522304/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826858.741085:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011522304/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826858.741096:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826858.741101:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826858.741105:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011522304:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826858.741116:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011522432:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826858.741138:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826858.741150:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826858/real 1787826858] req@ffff907da562cf00 x1874670011522432/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826858.741168:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011522432/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826858.741177:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826858.741180:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826858.741183:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011522432:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826858.741208:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011522560:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826858.741230:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826858.741241:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826858/real 1787826858] req@ffff907da562ed00 x1874670011522560/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826858.741251:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011522560/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826858.741259:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826858.741263:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826858.741266:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011522560:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826858.741272:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011522688:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826858.741294:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826858.741304:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826858/real 1787826858] req@ffff907da562c780 x1874670011522688/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826858.741315:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011522688/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826858.741322:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826858.741325:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826858.741328:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011522688:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826858.868720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787826858.868758:0:64869:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 2/2 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1787826858.868809:0:60482:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 2 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1787826858.868818:0:60482:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800028400000000 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1787826858.868823:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1787826858.868856:0:60482:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1f7:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1787826858.869926:0:60482:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1f7:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1787826858.869946:0:60482:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001b7e234c/0xe5f736940357b612 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400020000 nid: local remote: 0x7918213f4fc0f53c expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:0.0:1787826858.869956:0:60482:0:(vvp_io.c:1889:vvp_io_init()) [0x200000404:0x1fb:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1787826858.869960:0:60482:0:(vvp_io.c:304:vvp_io_fini()) [0x200000404:0x1fb:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1787826858.869995:0:60482:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x804869400000020 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826858.870004:0:60482:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001b7e234c/0xe5f736940357b612 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400020020 nid: local remote: 0x7918213f4fc0f53c expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826858.870010:0:60482:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1787826858.870039:0:60482:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff907d986d0000 lock: 0000000018d30c76/0xe5f736940357b532 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x304:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x804869400000020 nid: local remote: 0x7918213f4fc0f256 expref: -99 pid: 64727 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:0.0:1787826858.870059:0:60482:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff907d986d0000 lock: 000000001b7e234c/0xe5f736940357b612 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x306:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400020020 nid: local remote: 0x7918213f4fc0f53c expref: -99 pid: 65313 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1787826858.870069:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826858.870070:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826858.870073:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826858.870086:0:2349:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff907d908461c0 x1874670011522816/t0(0) o103->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826859.892737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826859.892775:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826860.916762:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826860.916831:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826861.940683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826861.940735:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826862.964723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826862.964962:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787826863.860772:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826863.860783:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787826863.860788:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826863.860798:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2267 00000100:00080000:3.0:1787826863.860810:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826863.860817:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826863.860862:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011522944/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826863.860902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826863.860905:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787826863.860906:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826863.860910:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2267 00000100:00080000:3.0:1787826863.860913:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826863.860915:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826863.860926:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826863.860936:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011523072/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826863.860944:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826863.860947:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787826863.860948:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826863.860953:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2267 00000100:00080000:3.0:1787826863.860955:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826863.860957:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826863.860960:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826863.860967:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011523200/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826863.860979:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826863.860982:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787826863.860984:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826863.860988:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2267 00000100:00080000:3.0:1787826863.860990:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826863.860993:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826863.860999:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826863.861006:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011522944:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826863.861019:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011523328/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787826863.861088:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826863.861122:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826863/real 1787826863] req@ffff907da562f480 x1874670011522944/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826863.861135:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011522944/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826863.861143:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826863.861150:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826863.861153:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011522944:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826863.861159:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011523072:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826863.861173:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826863.861180:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826863/real 1787826863] req@ffff907da562c000 x1874670011523072/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826863.861192:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011523072/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826863.861197:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826863.861199:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826863.861201:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011523072:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826863.861205:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011523200:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826863.861215:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826863.861220:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826863/real 1787826863] req@ffff907da562e580 x1874670011523200/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826863.861227:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011523200/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826863.861231:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826863.861233:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826863.861235:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011523200:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826863.861256:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011523328:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826863.861265:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826863.861271:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826863/real 1787826863] req@ffff907da562d2c0 x1874670011523328/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826863.861277:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011523328/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826863.861285:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826863.861287:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826863.861289:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011523328:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826863.988701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826863.988772:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826865.012747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826865.013029:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826866.036734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787826866.036836:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826867.061185:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826867.063577:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826868.084788:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826868.085334:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:0.0:1787826868.980718:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826868.980729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826868.980733:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826868.980744:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2272 00000100:00080000:0.0:1787826868.980755:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826868.980763:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826868.980797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011523456/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826868.980825:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826868.980828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826868.980829:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826868.980832:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2272 00000100:00080000:0.0:1787826868.980836:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826868.980840:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826868.980849:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826868.980861:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011523456:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787826868.980864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011523584/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826868.980876:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826868.980879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826868.980881:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826868.980886:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2272 00000100:00080000:0.0:1787826868.980890:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826868.980896:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826868.980901:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826868.980915:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011523712/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787826868.980921:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787826868.980930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826868.980934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826868.980936:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826868.980941:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2272 00000100:00080000:0.0:1787826868.980947:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826868.980951:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826868.980957:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826868.980970:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826868/real 1787826868] req@ffff907d90846580 x1874670011523456/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787826868.980977:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011523840/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826868.980998:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011523456/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826868.981009:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826868.981014:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826868.981018:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011523456:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826868.981038:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011523584:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826868.981060:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826868.981075:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826868/real 1787826868] req@ffff907d90847c00 x1874670011523584/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826868.981091:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011523584/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826868.981098:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826868.981100:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826868.981103:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011523584:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826868.981109:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011523712:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826868.981127:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826868.981137:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826868/real 1787826868] req@ffff907d908470c0 x1874670011523712/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826868.981148:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011523712/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826868.981155:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826868.981158:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826868.981162:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011523712:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826868.981167:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011523840:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826868.981196:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826868.981208:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826868/real 1787826868] req@ffff907d90845e00 x1874670011523840/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826868.981219:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011523840/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826868.981233:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826868.981236:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826868.981239:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011523840:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787826869.108780:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826869.108928:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826870.132838:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787826870.132875:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826871.156706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826871.156794:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826872.180755:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826872.180803:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826873.204729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787826873.204770:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826874.100730:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826874.100741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826874.100745:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826874.100760:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2277 00000100:00080000:0.0:1787826874.100770:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826874.100778:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826874.100821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011523968/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826874.100850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826874.100853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826874.100855:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826874.100859:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2277 00000100:00080000:0.0:1787826874.100862:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826874.100866:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826874.100875:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826874.100887:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011524096/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826874.100897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826874.100900:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826874.100902:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826874.100906:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2277 00000100:00080000:0.0:1787826874.100909:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826874.100913:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826874.100917:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826874.100928:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011524224/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826874.100943:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787826874.100943:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011523968:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787826874.100961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826874.100963:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826874.100968:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2277 00000100:00080000:0.0:1787826874.100975:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826874.100979:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826874.100985:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826874.100999:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011524352/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787826874.101028:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826874.101075:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826874/real 1787826874] req@ffff907d90844b40 x1874670011523968/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826874.101088:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011523968/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826874.101097:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826874.101100:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826874.101103:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011523968:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826874.101109:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011524096:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826874.101140:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826874.101153:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826874/real 1787826874] req@ffff907d908443c0 x1874670011524096/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826874.101166:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011524096/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826874.101179:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826874.101183:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826874.101186:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011524096:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826874.101216:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011524224:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826874.101236:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826874.101247:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826874/real 1787826874] req@ffff907d90844000 x1874670011524224/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826874.101267:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011524224/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826874.101276:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826874.101279:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826874.101283:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011524224:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826874.101290:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011524352:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826874.101322:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826874.101333:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826874/real 1787826874] req@ffff907d90844780 x1874670011524352/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826874.101356:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011524352/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826874.101365:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826874.101368:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826874.101372:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011524352:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826874.228744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826874.228772:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826875.252771:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826875.252851:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826876.276852:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826876.276956:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:0.0:1787826877.300794:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787826877.300917:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826878.324817:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826878.324868:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826879.221105:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826879.221119:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826879.221124:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826879.221136:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2282 00000100:00080000:2.0:1787826879.221149:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826879.221158:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826879.221204:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011524480/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826879.221230:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826879.221233:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826879.221234:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826879.221237:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2282 00000100:00080000:2.0:1787826879.221240:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826879.221242:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826879.221251:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826879.221271:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011524608/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826879.221285:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826879.221288:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826879.221290:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826879.221294:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2282 00000100:00080000:2.0:1787826879.221299:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787826879.221302:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011524480:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787826879.221306:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826879.221314:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826879.221328:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011524736/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826879.221344:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826879.221347:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826879.221349:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826879.221353:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2282 00000100:00080000:2.0:1787826879.221365:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826879.221368:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826879.221374:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787826879.221377:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787826879.221388:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011524864/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826879.221438:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826879/real 1787826879] req@ffff907d9114ad00 x1874670011524480/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826879.221461:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011524480/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826879.221471:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826879.221475:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826879.221479:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011524480:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826879.221485:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011524608:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826879.221508:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826879.221520:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826879/real 1787826879] req@ffff907d91149a40 x1874670011524608/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826879.221536:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011524608/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826879.221544:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826879.221548:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826879.221551:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011524608:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826879.221577:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011524736:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826879.221597:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826879.221638:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826879/real 1787826879] req@ffff907d9114a580 x1874670011524736/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826879.221650:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011524736/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826879.221658:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826879.221661:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826879.221664:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011524736:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826879.221670:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011524864:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826879.221695:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826879.221706:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826879/real 1787826879] req@ffff907d9114b840 x1874670011524864/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826879.221717:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011524864/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826879.221725:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826879.221728:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826879.221730:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011524864:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826879.348743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826879.348790:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826880.372726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787826880.372770:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826881.396751:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826881.396818:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826882.420738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826882.420807:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826883.444771:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826883.444838:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826884.340710:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826884.340721:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826884.340724:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826884.340732:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2288 00000100:00080000:2.0:1787826884.340741:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826884.340747:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826884.340775:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011524992/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826884.340797:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826884.340800:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826884.340802:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826884.340806:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2288 00000100:00080000:2.0:1787826884.340809:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826884.340813:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826884.340823:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826884.340835:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011525120/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826884.340846:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826884.340849:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826884.340851:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826884.340854:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2288 00000100:00080000:2.0:1787826884.340857:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826884.340860:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826884.340863:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826884.340874:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011525248/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826884.340884:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826884.340888:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826884.340891:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787826884.340892:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011524992:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787826884.340897:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2288 00000100:00080000:2.0:1787826884.340902:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826884.340906:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826884.340911:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826884.340924:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011525376/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787826884.340952:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826884.340983:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826884/real 1787826884] req@ffff907d91149680 x1874670011524992/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826884.340998:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011524992/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826884.341005:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826884.341007:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826884.341009:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011524992:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826884.341014:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011525120:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826884.341030:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826884.341041:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826884/real 1787826884] req@ffff907d9114bc00 x1874670011525120/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826884.341058:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011525120/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826884.341065:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826884.341069:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826884.341072:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011525120:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826884.341096:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011525248:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826884.341116:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826884.341127:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826884/real 1787826884] req@ffff907d911483c0 x1874670011525248/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826884.341138:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011525248/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826884.341152:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826884.341155:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826884.341158:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011525248:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826884.341163:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011525376:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826884.341181:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826884.341191:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826884/real 1787826884] req@ffff907d9114b480 x1874670011525376/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826884.341201:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011525376/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826884.341206:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826884.341212:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826884.341214:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011525376:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826884.468729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787826884.468788:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826885.492831:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826885.493273:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826886.516812:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826886.516964:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787826887.540711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826887.540770:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826888.564729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826888.564780:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787826889.461394:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826889.461404:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787826889.461408:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826889.461419:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2293 00000100:00080000:3.0:1787826889.461431:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826889.461439:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826889.461478:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011525504/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826889.461507:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826889.461511:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787826889.461512:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826889.461517:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2293 00000100:00080000:3.0:1787826889.461520:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826889.461524:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826889.461533:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826889.461545:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011525632/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826889.461556:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826889.461559:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787826889.461560:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826889.461567:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2293 00000100:00100000:1.0:1787826889.461569:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011525504:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787826889.461571:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826889.461579:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826889.461583:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826889.461603:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011525760/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826889.461651:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826889.461655:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787826889.461656:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826889.461660:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2293 00000100:00080000:3.0:1787826889.461664:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826889.461668:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826889.461673:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787826889.461674:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826889.461689:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011525888/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826889.461728:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826889/real 1787826889] req@ffff907da562c3c0 x1874670011525504/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826889.461746:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011525504/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826889.461760:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826889.461765:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826889.461769:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011525504:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826889.461777:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011525632:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826889.461817:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826889.461829:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826889/real 1787826889] req@ffff907da562f0c0 x1874670011525632/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826889.461846:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011525632/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826889.461854:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826889.461857:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826889.461861:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011525632:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826889.461882:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011525760:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826889.461899:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826889.461908:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826889/real 1787826889] req@ffff907da562fc00 x1874670011525760/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826889.461919:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011525760/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826889.461927:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826889.461929:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826889.461932:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011525760:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826889.461937:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011525888:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826889.461955:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826889.461963:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826889/real 1787826889] req@ffff907da562e940 x1874670011525888/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826889.461975:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011525888/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826889.461982:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826889.461985:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826889.461988:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011525888:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826889.588713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826889.588778:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826890.612831:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826890.613141:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787826891.636774:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826891.636881:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826892.660762:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826892.660838:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826893.684769:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826893.684831:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826894.580730:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826894.580739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826894.580744:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826894.580753:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2298 00000100:00080000:2.0:1787826894.580764:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826894.580772:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826894.580810:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011526016/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826894.580837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826894.580841:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826894.580843:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826894.580847:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2298 00000100:00080000:2.0:1787826894.580851:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826894.580855:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826894.580865:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826894.580879:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011526144/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826894.580890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826894.580893:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826894.580895:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826894.580899:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2298 00000100:00080000:2.0:1787826894.580902:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826894.580906:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826894.580911:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826894.580925:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011526016:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787826894.580926:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011526272/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826894.580951:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826894.580955:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826894.580956:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826894.580961:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2298 00000100:00080000:2.0:1787826894.580965:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826894.580969:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826894.580973:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826894.580987:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011526400/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787826894.581025:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826894.581082:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826894/real 1787826894] req@ffff907d9114b480 x1874670011526016/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826894.581102:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011526016/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826894.581114:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826894.581119:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826894.581124:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011526016:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826894.581132:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011526144:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826894.581170:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826894.581188:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826894/real 1787826894] req@ffff907d911483c0 x1874670011526144/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826894.581201:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011526144/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826894.581218:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826894.581221:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826894.581224:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011526144:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826894.581253:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011526272:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826894.581275:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826894.581287:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826894/real 1787826894] req@ffff907d9114bc00 x1874670011526272/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826894.581298:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011526272/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826894.581306:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826894.581310:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826894.581313:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011526272:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826894.581319:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011526400:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826894.581337:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826894.581347:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826894/real 1787826894] req@ffff907d91149680 x1874670011526400/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826894.581360:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011526400/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826894.581365:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826894.581367:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826894.581369:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011526400:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826894.708734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826894.708860:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826895.732708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826895.732772:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826896.756770:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826896.756843:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826897.780719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826897.780913:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826898.804714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826898.804783:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826899.700776:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826899.700788:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826899.700794:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826899.700808:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2303 00000100:00080000:0.0:1787826899.700822:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826899.700840:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826899.700900:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011526528/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826899.700938:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826899.700942:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826899.700944:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826899.700949:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2303 00000100:00080000:0.0:1787826899.700953:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826899.700958:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826899.700969:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826899.700984:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011526656/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826899.700995:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787826899.700998:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011526528:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787826899.700999:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826899.701004:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826899.701010:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2303 00000100:00080000:0.0:1787826899.701023:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826899.701027:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826899.701032:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826899.701041:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011526784/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826899.701048:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826899.701051:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826899.701052:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826899.701056:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2303 00000100:00080000:0.0:1787826899.701058:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826899.701060:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826899.701064:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787826899.701068:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787826899.701085:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011526912/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826899.701152:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826899/real 1787826899] req@ffff907d90845680 x1874670011526528/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826899.701168:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011526528/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826899.701180:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826899.701185:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826899.701189:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011526528:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826899.701215:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011526656:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826899.701240:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826899.701254:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826899/real 1787826899] req@ffff907d90846d00 x1874670011526656/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826899.701268:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011526656/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826899.701276:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826899.701279:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826899.701282:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011526656:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826899.701288:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011526784:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826899.701308:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826899.701319:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826899/real 1787826899] req@ffff907d908452c0 x1874670011526784/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826899.701331:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011526784/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826899.701339:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826899.701343:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826899.701346:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011526784:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826899.701352:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011526912:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826899.701369:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826899.701379:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826899/real 1787826899] req@ffff907d90845a40 x1874670011526912/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826899.701390:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011526912/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826899.701403:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826899.701406:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826899.701409:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011526912:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826899.828734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826899.828820:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826900.852708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826900.852765:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826901.876727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826901.876786:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826902.900796:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826902.900865:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826903.924725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787826903.924792:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787826904.820863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826904.820874:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787826904.820879:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826904.820894:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2308 00000100:00080000:3.0:1787826904.820909:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826904.820920:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826904.820966:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011527040/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826904.821252:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826904.821257:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787826904.821260:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826904.821265:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2308 00000100:00080000:3.0:1787826904.821270:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826904.821277:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826904.821292:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826904.821313:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011527168/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826904.821327:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826904.821331:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787826904.821334:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826904.821339:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2308 00000100:00080000:3.0:1787826904.821345:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826904.821350:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826904.821398:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826904.821409:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011527296/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826904.821419:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826904.821423:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787826904.821425:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826904.821428:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2308 00000100:00080000:3.0:1787826904.821432:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826904.821436:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826904.821441:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826904.821454:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011527424/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787826904.821518:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011527040:192.168.203.104@tcp:250:kworker.0 00000800:00000100:2.0:1787826904.821640:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787826904.821693:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826904/real 1787826904] req@ffff907da562e1c0 x1874670011527040/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787826904.821726:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011527040/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787826904.821738:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787826904.821743:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787826904.821747:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011527040:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787826904.821754:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011527168:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787826904.821777:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787826904.821790:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826904/real 1787826904] req@ffff907da562f840 x1874670011527168/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787826904.821799:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011527168/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787826904.821804:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787826904.821807:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787826904.821809:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011527168:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787826904.821815:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011527296:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787826904.821835:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787826904.821846:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826904/real 1787826904] req@ffff907da562de00 x1874670011527296/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787826904.821857:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011527296/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787826904.821865:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787826904.821868:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787826904.821871:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011527296:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787826904.821878:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011527424:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787826904.821896:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787826904.821913:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826904/real 1787826904] req@ffff907da562cb40 x1874670011527424/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787826904.821929:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011527424/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787826904.821937:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787826904.821940:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787826904.821943:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011527424:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826904.948727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826904.948797:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826905.972733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826905.972812:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826906.996741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826906.996820:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826908.020892:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826908.021249:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787826909.044774:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826909.044849:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826909.940840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826909.940854:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826909.940860:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826909.940874:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2313 00000100:00080000:2.0:1787826909.940889:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826909.940898:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826909.940951:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011527552/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826909.940988:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826909.940992:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826909.940994:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826909.940998:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2313 00000100:00080000:2.0:1787826909.941002:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826909.941006:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826909.941018:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826909.941034:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011527680/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826909.941037:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011527552:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787826909.941066:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826909.941069:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826909.941071:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826909.941075:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2313 00000100:00080000:2.0:1787826909.941079:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826909.941083:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826909.941090:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826909.941102:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011527808/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826909.941111:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826909.941115:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826909.941116:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826909.941121:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2313 00000100:00080000:2.0:1787826909.941124:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826909.941128:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:1.0:1787826909.941134:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:2.0:1787826909.941134:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826909.941149:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011527936/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826909.941179:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826909/real 1787826909] req@ffff907d9114b840 x1874670011527552/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826909.941199:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011527552/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826909.941211:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826909.941215:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826909.941219:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011527552:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826909.941245:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011527680:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826909.941268:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826909.941281:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826909/real 1787826909] req@ffff907d9114a580 x1874670011527680/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826909.941293:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011527680/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826909.941302:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826909.941305:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826909.941309:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011527680:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826909.941314:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011527808:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826909.941334:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826909.941345:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826909/real 1787826909] req@ffff907d91149a40 x1874670011527808/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826909.941356:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011527808/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826909.941364:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826909.941367:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826909.941371:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011527808:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826909.941376:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011527936:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826909.941394:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826909.941404:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826909/real 1787826909] req@ffff907d9114ad00 x1874670011527936/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826909.941415:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011527936/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826909.941423:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826909.941427:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826909.941430:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011527936:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826910.068759:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826910.068788:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826911.092719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787826911.092765:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826912.116704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826912.116762:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826913.140748:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826913.140816:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826914.164705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826914.164758:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826915.060713:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826915.060721:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826915.060725:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826915.060734:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2318 00000100:00080000:2.0:1787826915.060745:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826915.060752:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826915.060799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011528064/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826915.060836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826915.060840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826915.060842:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826915.060846:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2318 00000100:00080000:2.0:1787826915.060850:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826915.060855:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826915.060865:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826915.060875:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011528064:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787826915.060881:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011528192/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826915.060891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826915.060894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826915.060896:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826915.060900:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2318 00000100:00080000:2.0:1787826915.060903:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826915.060908:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826915.060913:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826915.060925:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011528320/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826915.060934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826915.060937:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826915.060939:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826915.060943:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2318 00000100:00080000:2.0:1787826915.060947:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826915.060950:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826915.060956:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826915.060967:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011528448/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826915.060971:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011528192:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826915.061023:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011528320:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826915.061054:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011528448:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826915.188709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826915.188773:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826916.212731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826916.212821:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826917.236757:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826917.236824:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787826918.132866:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:0.0:1787826918.132908:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787826918.132924:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826918.132957:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826918.132967:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787826918.132985:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:1.0:1787826918.133007:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826915/real 1787826918] req@ffff907d9114a940 x1874670011528192/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826918.133031:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011528192/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826918.133043:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826918.133047:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826918.133052:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011528192:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826918.133058:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826915/real 1787826918] req@ffff907d911492c0 x1874670011528320/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826918.133070:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011528320/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826918.133078:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826918.133082:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826918.133085:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011528320:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826918.133091:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826915/real 1787826918] req@ffff907d91148f00 x1874670011528064/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826918.133103:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011528064/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826918.133119:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826918.133123:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826918.133126:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011528064:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826918.133147:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826915/real 1787826918] req@ffff907d91148b40 x1874670011528448/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826918.133159:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011528448/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826918.133166:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826918.133169:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826918.133173:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011528448:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826918.260696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826918.260750:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826919.284757:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826919.284814:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826920.180705:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826920.180715:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826920.180719:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826920.180729:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2323 00000100:00080000:2.0:1787826920.180740:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826920.180747:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826920.180776:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011528576/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826920.180798:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826920.180801:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826920.180805:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826920.180808:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2323 00000100:00080000:2.0:1787826920.180810:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826920.180813:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826920.180821:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826920.180831:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011528704/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826920.180848:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011528576:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787826920.180849:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826920.180858:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826920.180860:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826920.180864:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2323 00000100:00080000:2.0:1787826920.180868:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826920.180873:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826920.180878:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826920.180891:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011528832/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826920.180900:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826920.180904:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826920.180906:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826920.180909:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2323 00000100:00080000:2.0:1787826920.180913:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787826920.180916:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787826920.180921:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826920.180928:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826920.180939:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011528960/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826920.180958:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826920/real 1787826920] req@ffff907d91148b40 x1874670011528576/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826920.180987:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011528576/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826920.180996:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826920.181000:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826920.181003:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011528576:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826920.181026:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011528704:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826920.181046:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826920.181057:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826920/real 1787826920] req@ffff907d91148f00 x1874670011528704/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826920.181067:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011528704/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826920.181073:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826920.181076:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826920.181079:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011528704:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826920.181084:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011528832:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826920.181100:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826920.181109:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826920/real 1787826920] req@ffff907d911492c0 x1874670011528832/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826920.181118:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011528832/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826920.181125:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826920.181128:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826920.181130:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011528832:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826920.181135:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011528960:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826920.181151:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826920.181162:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826920/real 1787826920] req@ffff907d9114a940 x1874670011528960/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826920.181172:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011528960/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826920.181180:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826920.181183:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826920.181186:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011528960:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826920.308694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826920.308738:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826921.332706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826921.332763:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826922.356732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826922.356851:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826923.380741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826923.380815:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826924.404720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826924.404761:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826925.300736:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826925.300746:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826925.300750:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826925.300759:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2329 00000100:00080000:0.0:1787826925.300769:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826925.300780:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826925.300916:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011529088/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826925.300992:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011529088:192.168.203.104@tcp:250:kworker.0 00000800:00000100:1.0:1787826925.301051:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787826925.301079:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826925.301082:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826925.301084:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826925.301089:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2329 00000100:00080000:0.0:1787826925.301092:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826925.301097:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826925.301106:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826925.301118:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011529216/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826925.301174:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826925/real 1787826925] req@ffff907d90844f00 x1874670011529088/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826925.301190:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011529088/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826925.301200:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826925.301204:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826925.301208:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011529088:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826925.301229:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011529216:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826925.301250:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826925.301274:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826925/real 1787826925] req@ffff907d90846940 x1874670011529216/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826925.301285:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011529216/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826925.301293:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826925.301295:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826925.301298:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011529216:192.168.203.104@tcp:38:kworker.0 00000100:00080000:0.0:1787826925.301347:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826925.301351:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826925.301353:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826925.301357:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2329 00000100:00080000:0.0:1787826925.301360:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826925.301364:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826925.301370:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826925.301387:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011529344/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826925.301401:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826925.301404:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826925.301406:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826925.301409:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2329 00000100:00080000:0.0:1787826925.301412:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826925.301415:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826925.301419:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826925.301428:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011529472/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826925.301482:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011529344:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826925.301503:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826925.301514:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826925/real 1787826925] req@ffff907d90847480 x1874670011529344/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826925.301529:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011529344/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826925.301537:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826925.301539:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826925.301542:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011529344:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826925.301547:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011529472:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826925.301562:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826925.301571:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826925/real 1787826925] req@ffff907d90846940 x1874670011529472/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826925.301580:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011529472/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826925.301586:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826925.301589:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826925.301592:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011529472:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826925.428716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826925.428796:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826926.452716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826926.452803:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826927.476751:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826927.476819:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826928.500737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826928.500823:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826929.524748:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826929.525042:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826930.420753:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826930.420766:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826930.420772:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826930.420784:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2334 00000100:00080000:0.0:1787826930.420797:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826930.420804:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826930.420849:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011529600/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826930.420882:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826930.420886:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826930.420888:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826930.420893:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2334 00000100:00080000:0.0:1787826930.420897:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826930.420901:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826930.420928:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826930.420943:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011529728/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826930.420965:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826930.420970:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826930.420974:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787826930.420978:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011529600:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787826930.420982:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2334 00000100:00080000:0.0:1787826930.420989:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826930.420996:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826930.421002:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826930.421014:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011529856/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826930.421024:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826930.421028:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826930.421030:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826930.421035:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2334 00000100:00080000:0.0:1787826930.421043:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826930.421048:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826930.421054:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787826930.421060:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787826930.421070:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011529984/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826930.421107:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826930/real 1787826930] req@ffff907d90844f00 x1874670011529600/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826930.421123:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011529600/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826930.421133:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826930.421138:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826930.421142:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011529600:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826930.421164:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011529728:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826930.421188:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826930.421201:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826930/real 1787826930] req@ffff907d90845a40 x1874670011529728/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826930.421213:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011529728/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826930.421221:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826930.421224:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826930.421226:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011529728:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826930.421231:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011529856:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826930.421262:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826930.421274:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826930/real 1787826930] req@ffff907d908452c0 x1874670011529856/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826930.421285:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011529856/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826930.421299:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826930.421302:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826930.421305:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011529856:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826930.421318:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011529984:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826930.421339:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826930.421350:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826930/real 1787826930] req@ffff907d90846d00 x1874670011529984/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826930.421361:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011529984/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826930.421369:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826930.421372:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826930.421375:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011529984:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787826930.548786:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826930.549519:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787826931.573601:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826931.573765:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826932.596774:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826932.596947:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826933.620701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826933.620761:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826934.644712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826934.644765:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826935.541000:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826935.541011:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826935.541015:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826935.541026:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2339 00000100:00080000:2.0:1787826935.541037:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826935.541044:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826935.541083:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011530112/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826935.541109:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826935.541112:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826935.541114:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826935.541119:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2339 00000100:00080000:2.0:1787826935.541122:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826935.541125:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826935.541136:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826935.541149:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011530240/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826935.541159:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011530112:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787826935.541171:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826935.541174:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826935.541175:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826935.541180:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2339 00000100:00080000:2.0:1787826935.541183:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826935.541186:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826935.541192:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826935.541203:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011530368/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826935.541212:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826935.541216:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826935.541218:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826935.541221:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2339 00000100:00080000:2.0:1787826935.541225:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787826935.541229:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787826935.541230:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826935.541238:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826935.541250:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011530496/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826935.541267:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826935/real 1787826935] req@ffff907d9114ad00 x1874670011530112/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826935.541281:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011530112/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826935.541289:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826935.541292:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826935.541295:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011530112:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826935.541312:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011530240:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826935.541326:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826935.541336:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826935/real 1787826935] req@ffff907d91149a40 x1874670011530240/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826935.541348:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011530240/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826935.541356:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826935.541359:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826935.541362:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011530240:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826935.541368:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011530368:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826935.541387:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826935.541397:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826935/real 1787826935] req@ffff907d9114a580 x1874670011530368/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826935.541405:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011530368/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826935.541410:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826935.541412:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826935.541414:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011530368:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826935.541418:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011530496:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826935.541429:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826935.541435:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826935/real 1787826935] req@ffff907d9114b840 x1874670011530496/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826935.541442:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011530496/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826935.541446:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826935.541448:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826935.541450:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011530496:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826935.668737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826935.668742:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826936.692752:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826936.692823:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826937.716776:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787826937.716826:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826938.740786:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826938.740907:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826939.765033:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826939.766966:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:2.0:1787826940.660728:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826940.660739:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826940.660744:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826940.660753:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2344 00000100:00080000:2.0:1787826940.660763:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826940.660771:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826940.660825:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011530624/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826940.660866:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826940.660869:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826940.660871:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826940.660876:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2344 00000100:00080000:2.0:1787826940.660879:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826940.660883:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826940.660893:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826940.660910:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011530752/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826940.660922:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826940.660925:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826940.660927:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826940.660931:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2344 00000100:00080000:2.0:1787826940.660935:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826940.660939:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826940.660945:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826940.660956:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011530880/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826940.660974:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011530624:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787826940.660975:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826940.660983:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826940.660985:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826940.660991:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2344 00000100:00080000:2.0:1787826940.660996:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826940.661000:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826940.661005:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826940.661016:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011531008/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787826940.661190:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826940.661258:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826940/real 1787826940] req@ffff907d91149680 x1874670011530624/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826940.661284:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011530624/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826940.661296:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826940.661300:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826940.661306:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011530624:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826940.661315:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011530752:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826940.661347:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826940.661367:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826940/real 1787826940] req@ffff907d9114bc00 x1874670011530752/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826940.661388:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011530752/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826940.661657:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826940.661664:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826940.661668:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011530752:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826940.661705:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011530880:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826940.661734:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826940.661758:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826940/real 1787826940] req@ffff907d911483c0 x1874670011530880/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826940.661785:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011530880/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826940.661797:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826940.661801:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826940.661805:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011530880:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826940.661815:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011531008:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826940.661851:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826940.661863:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826940/real 1787826940] req@ffff907d9114b480 x1874670011531008/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826940.661920:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011531008/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826940.661929:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826940.661932:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826940.661935:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011531008:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787826940.788794:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826940.788912:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787826941.812727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826941.812812:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826942.836736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826942.836957:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826943.861053:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826943.861658:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787826944.884983:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826944.885913:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:3.0:1787826945.780852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826945.780864:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787826945.780870:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826945.780882:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2349 00000100:00080000:3.0:1787826945.780896:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826945.780903:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826945.780952:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011531136/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826945.780989:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826945.780993:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787826945.780995:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826945.781000:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2349 00000100:00080000:3.0:1787826945.781004:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826945.781008:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826945.781019:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826945.781036:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011531264/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826945.781047:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826945.781050:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787826945.781052:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826945.781056:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2349 00000100:00080000:3.0:1787826945.781059:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826945.781063:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826945.781068:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826945.781079:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011531392/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826945.781088:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826945.781091:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787826945.781093:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826945.781096:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2349 00000100:00080000:3.0:1787826945.781100:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826945.781103:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826945.781108:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826945.781117:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011531520/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826945.782651:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011531136:192.168.203.104@tcp:250:kworker.0 00000800:00000100:1.0:1787826945.782733:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826945.782792:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826945/real 1787826945] req@ffff907da562da40 x1874670011531136/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826945.782824:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011531136/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826945.782845:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826945.782850:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826945.782854:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011531136:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826945.782861:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011531264:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826945.782885:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826945.782898:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826945/real 1787826945] req@ffff907da562cb40 x1874670011531264/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826945.782908:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011531264/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826945.782916:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826945.782919:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826945.782922:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011531264:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826945.782929:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011531392:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826945.782949:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826945.782974:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826945/real 1787826945] req@ffff907da562de00 x1874670011531392/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826945.782985:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011531392/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826945.782992:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826945.782994:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826945.782997:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011531392:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826945.783003:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011531520:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826945.783033:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826945.783054:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826945/real 1787826945] req@ffff907da562f840 x1874670011531520/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826945.783066:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011531520/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826945.783073:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826945.783075:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826945.783078:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011531520:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787826945.908779:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826945.910859:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826946.933829:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826946.933914:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826947.956894:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826947.957067:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826948.980819:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787826948.980927:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:0.0:1787826950.004717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826950.005838:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787826950.901233:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826950.901253:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787826950.901257:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826950.901268:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2354 00000100:00080000:3.0:1787826950.901279:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826950.901295:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826950.901332:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011531648/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826950.901362:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826950.901366:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787826950.901368:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826950.901372:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2354 00000100:00080000:3.0:1787826950.901375:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826950.901378:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826950.901387:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826950.901411:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011531776/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826950.901422:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826950.901424:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787826950.901426:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826950.901430:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2354 00000100:00080000:3.0:1787826950.901433:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826950.901436:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826950.901441:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826950.901450:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011531904/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826950.901463:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826950.901466:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787826950.901467:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826950.901470:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2354 00000100:00080000:3.0:1787826950.901473:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826950.901476:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826950.901480:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826950.901491:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011532032/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826950.901636:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011531648:192.168.203.104@tcp:250:kworker.0 00000800:00000100:1.0:1787826950.901689:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826950.901736:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826950/real 1787826950] req@ffff907da562e1c0 x1874670011531648/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826950.901748:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011531648/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826950.901755:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826950.901758:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826950.901771:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011531648:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826950.901777:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011531776:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826950.901795:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826950.901803:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826950/real 1787826950] req@ffff907da562e940 x1874670011531776/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826950.901815:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011531776/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826950.901823:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826950.901826:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826950.901828:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011531776:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826950.901834:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011531904:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826950.901855:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826950.901866:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826950/real 1787826950] req@ffff907da562fc00 x1874670011531904/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826950.901878:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011531904/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826950.901883:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826950.901886:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826950.901889:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011531904:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826950.901893:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011532032:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826950.901908:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826950.901914:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826950/real 1787826950] req@ffff907da562f0c0 x1874670011532032/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826950.901925:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011532032/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826950.901929:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826950.901931:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826950.901932:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011532032:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826951.028753:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826951.028837:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826952.052801:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787826952.053001:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826953.076818:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826953.076943:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787826954.100878:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826954.101880:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787826955.124694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826955.124906:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826956.020754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826956.020764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826956.020769:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826956.020779:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2359 00000100:00080000:0.0:1787826956.020791:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826956.020798:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826956.020836:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011532160/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826956.020864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826956.020868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826956.020870:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826956.020874:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2359 00000100:00080000:0.0:1787826956.020878:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826956.020881:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826956.020891:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826956.020907:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011532160:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787826956.020908:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011532288/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826956.020923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826956.020927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826956.020929:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826956.020932:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2359 00000100:00080000:0.0:1787826956.020936:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826956.020940:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826956.020946:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826956.020956:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011532416/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826956.020964:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787826956.020967:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787826956.020971:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826956.020974:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826956.020979:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2359 00000100:00080000:0.0:1787826956.020983:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826956.020987:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826956.020992:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826956.021001:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011532544/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826956.021012:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826956/real 1787826956] req@ffff907d90845680 x1874670011532160/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826956.021025:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011532160/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826956.021034:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826956.021039:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826956.021049:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011532160:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826956.021074:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011532288:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826956.021094:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826956.021103:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826956/real 1787826956] req@ffff907d90844780 x1874670011532288/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826956.021111:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011532288/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826956.021117:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826956.021119:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826956.021121:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011532288:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826956.021125:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011532416:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826956.021136:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826956.021142:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826956/real 1787826956] req@ffff907d90844000 x1874670011532416/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826956.021149:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011532416/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826956.021154:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826956.021155:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826956.021157:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011532416:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826956.021160:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011532544:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826956.021169:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826956.021173:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826956/real 1787826956] req@ffff907d908443c0 x1874670011532544/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826956.021179:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011532544/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826956.021188:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826956.021190:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826956.021192:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011532544:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826956.148792:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826956.148852:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787826957.172791:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787826957.172857:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826958.196752:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787826958.196825:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826959.220742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826959.220793:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826960.244782:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787826960.245019:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787826961.140755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826961.140764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787826961.140768:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826961.140778:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2364 00000100:00080000:3.0:1787826961.140788:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826961.140795:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787826961.140826:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011532672/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826961.140848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826961.140851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787826961.140853:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826961.140856:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2364 00000100:00080000:3.0:1787826961.140859:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826961.140863:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826961.140872:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787826961.140881:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011532800/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826961.140891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787826961.140896:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011532672:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787826961.140897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787826961.140900:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826961.140907:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2364 00000100:00080000:3.0:1787826961.140909:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826961.140912:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826961.140916:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826961.140925:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011532928/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787826961.140932:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787826961.140935:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787826961.140937:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787826961.140940:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2364 00000100:00080000:3.0:1787826961.140944:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787826961.140947:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787826961.140952:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787826961.140958:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826961.140963:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011533056/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826961.140994:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826961/real 1787826961] req@ffff907da562c3c0 x1874670011532672/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826961.141009:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011532672/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826961.141019:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826961.141024:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826961.141028:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011532672:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826961.141050:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011532800:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826961.141073:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826961.141085:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826961/real 1787826961] req@ffff907da562d2c0 x1874670011532800/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826961.141097:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011532800/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826961.141111:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826961.141114:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826961.141117:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011532800:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826961.141122:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011532928:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826961.141142:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826961.141153:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826961/real 1787826961] req@ffff907da562e580 x1874670011532928/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826961.141163:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011532928/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826961.141173:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826961.141175:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826961.141178:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011532928:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826961.141184:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011533056:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826961.141200:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826961.141209:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826961/real 1787826961] req@ffff907da562c000 x1874670011533056/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826961.141219:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011533056/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826961.141227:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826961.141229:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826961.141233:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011533056:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787826961.269171:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826961.269254:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826962.292776:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787826962.292810:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826963.316722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787826963.316760:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826964.340863:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826964.341030:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787826965.364751:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787826965.364773:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826966.260831:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826966.260849:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826966.260855:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826966.260867:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2370 00000100:00080000:2.0:1787826966.260879:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826966.260886:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826966.260934:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011533184/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826966.260960:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826966.260963:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826966.260965:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826966.260969:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2370 00000100:00080000:2.0:1787826966.260972:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826966.260977:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826966.260988:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826966.261002:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011533312/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826966.261013:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826966.261018:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787826966.261020:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011533184:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787826966.261025:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826966.261032:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2370 00000100:00080000:2.0:1787826966.261036:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826966.261041:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826966.261046:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826966.261059:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011533440/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826966.261068:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826966.261072:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826966.261074:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826966.261079:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2370 00000100:00080000:2.0:1787826966.261082:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826966.261086:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826966.261091:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787826966.261109:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787826966.261111:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011533568/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787826966.261172:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826966/real 1787826966] req@ffff907d9114b480 x1874670011533184/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826966.261200:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011533184/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826966.261214:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826966.261217:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826966.261220:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011533184:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826966.261254:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011533312:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787826966.261280:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826966.261303:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826966/real 1787826966] req@ffff907d911483c0 x1874670011533312/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826966.261316:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011533312/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826966.261321:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826966.261324:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826966.261326:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011533312:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787826966.261330:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011533440:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826966.261348:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826966.261357:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826966/real 1787826966] req@ffff907d9114bc00 x1874670011533440/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826966.261368:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011533440/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826966.261376:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826966.261379:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826966.261382:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011533440:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826966.261388:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011533568:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787826966.261407:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787826966.261419:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826966/real 1787826966] req@ffff907d91149680 x1874670011533568/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787826966.261430:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011533568/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787826966.261438:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787826966.261451:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787826966.261455:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011533568:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787826966.388750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787826966.388783:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826967.412718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787826967.412770:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826968.436780:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787826968.436882:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826969.460728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787826969.460792:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826970.484754:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787826970.484783:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826971.380736:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826971.380743:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826971.380747:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826971.380756:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2375 00000100:00080000:1.0:1787826971.380766:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826971.380773:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826971.380809:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011533696/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826971.380843:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826971.380847:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826971.380849:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826971.380853:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2375 00000100:00080000:1.0:1787826971.380857:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826971.380861:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826971.380871:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826971.380887:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011533824/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826971.380896:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826971.380900:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826971.380902:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826971.380906:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2375 00000100:00080000:1.0:1787826971.380910:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826971.380918:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826971.380924:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787826971.380924:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011533696:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787826971.380941:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011533952/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826971.380959:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826971.380962:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826971.380964:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826971.380969:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2375 00000100:00080000:1.0:1787826971.380978:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826971.380982:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826971.380989:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826971.381002:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011534080/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787826971.381023:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826971.381084:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826971/real 1787826971] req@ffff907d908a0f00 x1874670011533696/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826971.381107:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011533696/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787826996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826971.381120:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826971.381125:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826971.381129:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011533696:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826971.381160:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011533824:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826971.381187:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826971.381201:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826971/real 1787826971] req@ffff907d908a1680 x1874670011533824/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826971.381214:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011533824/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787826996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826971.381223:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826971.381237:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826971.381241:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011533824:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826971.381248:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011533952:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826971.381271:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826971.381283:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826971/real 1787826971] req@ffff907d908a12c0 x1874670011533952/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826971.381312:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011533952/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826971.381321:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826971.381324:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826971.381327:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011533952:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826971.381334:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011534080:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826971.381354:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826971.381365:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826971/real 1787826971] req@ffff907d908a21c0 x1874670011534080/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826971.381393:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011534080/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787826996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826971.381402:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826971.381406:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826971.381409:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011534080:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787826971.508775:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787826971.508847:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826972.532717:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826972.532744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787826973.556703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787826973.556724:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787826974.580994:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787826974.581691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787826975.604758:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787826975.605234:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826976.500687:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826976.500695:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826976.500698:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826976.500705:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2380 00000100:00080000:1.0:1787826976.500714:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826976.500719:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826976.500754:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011534208/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826976.500772:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826976.500775:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826976.500776:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826976.500779:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2380 00000100:00080000:1.0:1787826976.500782:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826976.500785:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826976.500794:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826976.500805:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011534336/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826976.500816:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787826976.500822:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011534208:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787826976.500828:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826976.500831:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826976.500835:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2380 00000100:00080000:1.0:1787826976.500839:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826976.500843:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826976.500847:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826976.500859:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011534464/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826976.500869:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826976.500872:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826976.500874:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826976.500877:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2380 00000100:00080000:1.0:1787826976.500881:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787826976.500886:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787826976.500889:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826976.500894:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826976.500902:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011534592/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826976.500926:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826976/real 1787826976] req@ffff907d908a2580 x1874670011534208/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826976.500943:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011534208/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826976.500953:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826976.500957:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826976.500962:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011534208:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826976.500983:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011534336:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826976.501003:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826976.501015:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826976/real 1787826976] req@ffff907d908a3840 x1874670011534336/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826976.501027:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011534336/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826976.501036:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826976.501039:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826976.501043:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011534336:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826976.501049:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011534464:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826976.501078:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826976.501091:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826976/real 1787826976] req@ffff907d908a2940 x1874670011534464/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826976.501108:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011534464/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826976.501174:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826976.501178:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826976.501182:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011534464:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826976.501189:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011534592:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826976.501211:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826976.501328:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826976/real 1787826976] req@ffff907d908a1e00 x1874670011534592/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826976.501343:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011534592/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826976.501387:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826976.501392:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826976.501397:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011534592:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826976.628699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826976.628776:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826977.652717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787826977.652817:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826978.676749:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787826978.676781:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826979.700749:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787826979.700772:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826980.724673:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826980.724743:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787826981.620769:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826981.620779:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787826981.620782:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826981.620791:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2385 00000100:00080000:1.0:1787826981.620802:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826981.620809:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787826981.620845:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011534720/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826981.620865:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826981.620868:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787826981.620869:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826981.620872:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2385 00000100:00080000:1.0:1787826981.620874:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826981.620876:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826981.620884:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787826981.620891:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011534848/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787826981.620903:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826981.620906:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787826981.620908:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826981.620912:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2385 00000100:00080000:1.0:1787826981.620915:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826981.620920:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826981.620925:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826981.620939:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011534976/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826981.620959:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011534720:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787826981.620963:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787826981.620973:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787826981.620976:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787826981.620979:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2385 00000100:00080000:1.0:1787826981.620982:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787826981.620984:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787826981.620988:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787826981.620999:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011535104/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826981.621100:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011534848:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826981.621144:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011534976:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826981.621166:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011535104:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787826981.748735:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826981.748815:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787826982.772762:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826982.772837:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826983.796728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826983.796835:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787826984.692908:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787826984.692952:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787826984.692968:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787826984.693018:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787826984.693027:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787826984.693035:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:3.0:1787826984.694597:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826981/real 1787826984] req@ffff907d908a1e00 x1874670011534976/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826984.694671:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011534976/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826984.694682:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826984.694687:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826984.694691:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011534976:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826984.694698:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826981/real 1787826984] req@ffff907d908a2940 x1874670011535104/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826984.694711:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011535104/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826984.694961:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826984.694965:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826984.694969:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011535104:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826984.694976:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826981/real 1787826984] req@ffff907d908a2d00 x1874670011534720/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826984.694989:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011534720/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826984.694996:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826984.695000:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826984.695002:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011534720:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826984.695009:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826981/real 1787826984] req@ffff907d908a0000 x1874670011534848/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826984.695020:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011534848/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826984.695028:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826984.695031:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826984.695034:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011534848:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826984.820761:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787826984.821646:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826985.844769:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787826985.845336:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787826986.740763:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826986.740776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787826986.740781:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826986.740796:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2390 00000100:00080000:0.0:1787826986.740808:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826986.740817:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787826986.740861:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011535232/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826986.740893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826986.740897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787826986.740899:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826986.740903:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2390 00000100:00080000:0.0:1787826986.740907:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826986.740911:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826986.740922:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787826986.740936:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011535360/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826986.740947:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826986.740951:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787826986.740954:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787826986.740957:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011535232:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787826986.740960:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2390 00000100:00080000:0.0:1787826986.740967:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826986.740971:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826986.740977:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826986.740988:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011535488/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787826986.740998:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787826986.741002:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787826986.741014:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787826986.741018:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2390 00000100:00080000:0.0:1787826986.741022:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787826986.741026:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787826986.741031:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787826986.741042:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011535616/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787826986.741060:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826986.741138:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826986/real 1787826986] req@ffff907d90844b40 x1874670011535232/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826986.741157:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011535232/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826986.741168:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826986.741172:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826986.741176:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011535232:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826986.741213:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011535360:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826986.741251:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826986.741267:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826986/real 1787826986] req@ffff907d90845e00 x1874670011535360/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826986.741278:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011535360/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826986.741292:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826986.741295:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826986.741298:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011535360:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826986.741308:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011535488:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826986.741328:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826986.741339:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826986/real 1787826986] req@ffff907d908470c0 x1874670011535488/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826986.741349:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011535488/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826986.741356:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826986.741358:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826986.741362:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011535488:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826986.741367:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011535616:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826986.741385:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826986.741395:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826986/real 1787826986] req@ffff907d90847c00 x1874670011535616/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826986.741405:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011535616/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826986.741411:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826986.741415:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826986.741418:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011535616:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787826986.868831:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787826986.868843:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787826987.892704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826987.892855:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787826988.916827:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826988.916879:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787826989.940737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787826989.940753:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826990.964723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826990.964843:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826991.860739:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826991.860749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826991.860754:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826991.860764:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2395 00000100:00080000:2.0:1787826991.860774:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826991.860789:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826991.860825:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011535744/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826991.860850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826991.860854:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826991.860855:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826991.860859:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2395 00000100:00080000:2.0:1787826991.860863:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826991.860866:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826991.860874:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826991.860887:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011535872/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826991.860897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826991.860900:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826991.860902:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826991.860906:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2395 00000100:00080000:2.0:1787826991.860910:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787826991.860914:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011535744:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787826991.860916:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826991.860926:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826991.860938:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011536000/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826991.860947:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826991.860950:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826991.860952:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826991.860955:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2395 00000100:00080000:2.0:1787826991.860972:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826991.860976:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826991.860982:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787826991.860982:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787826991.860996:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011536128/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826991.861025:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826991/real 1787826991] req@ffff907d9114b840 x1874670011535744/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826991.861043:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011535744/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826991.861053:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826991.861057:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826991.861060:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011535744:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826991.861072:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011535872:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826991.861096:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826991.861117:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826991/real 1787826991] req@ffff907d9114a580 x1874670011535872/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826991.861131:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011535872/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826991.861138:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826991.861141:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826991.861144:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011535872:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826991.861168:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011536000:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826991.861188:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826991.861201:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826991/real 1787826991] req@ffff907d91149a40 x1874670011536000/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826991.861212:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011536000/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826991.861220:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826991.861222:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826991.861225:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011536000:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826991.861231:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011536128:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826991.861250:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826991.861260:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826991/real 1787826991] req@ffff907d9114ad00 x1874670011536128/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826991.861271:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011536128/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826991.861278:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826991.861281:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826991.861284:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011536128:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826991.988759:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787826991.988761:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826993.012759:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787826993.012783:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826994.036700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787826994.036748:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826995.060741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826995.060849:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826996.084740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787826996.084746:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787826996.980755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826996.980779:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787826996.980786:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826996.980798:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2400 00000100:00080000:2.0:1787826996.980812:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826996.980821:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787826996.980888:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011536256/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826996.980928:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826996.980932:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787826996.980934:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826996.980939:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2400 00000100:00080000:2.0:1787826996.980942:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826996.980947:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826996.980958:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787826996.980971:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011536384/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826996.980980:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826996.980983:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787826996.980985:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826996.980988:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2400 00000100:00080000:2.0:1787826996.980992:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826996.980995:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826996.981000:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826996.981009:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011536512/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787826996.981018:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787826996.981020:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787826996.981022:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787826996.981024:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2400 00000100:00080000:2.0:1787826996.981027:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787826996.981030:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787826996.981033:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787826996.981040:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011536640/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787826996.982415:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011536256:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787826996.982517:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826996.982583:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826996/real 1787826996] req@ffff907d9114a940 x1874670011536256/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826996.982603:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011536256/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826996.982660:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826996.982665:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826996.982669:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011536256:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787826996.982676:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011536384:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787826996.982698:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826996.982710:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826996/real 1787826996] req@ffff907d911492c0 x1874670011536384/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826996.982720:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011536384/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826996.982727:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826996.982730:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826996.982733:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011536384:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787826996.982739:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011536512:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826996.982779:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826996.982803:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826996/real 1787826996] req@ffff907d91148f00 x1874670011536512/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826996.982815:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011536512/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826996.982822:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826996.982825:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826996.982828:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011536512:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826996.982834:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011536640:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787826996.982854:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787826996.982864:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787826996/real 1787826996] req@ffff907d91148b40 x1874670011536640/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787826996.982875:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011536640/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787826996.982883:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787826996.982886:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787826996.982889:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011536640:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787826997.108734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787826997.108764:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826998.132723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787826998.132837:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787826999.156739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787826999.156798:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827000.180720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827000.180759:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827001.204788:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827001.204795:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:2.0:1787827002.100948:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827002.100958:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827002.100962:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827002.100973:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2405 00000100:00080000:2.0:1787827002.100985:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827002.100992:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827002.101165:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011536768/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827002.101192:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827002.101196:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827002.101197:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827002.101202:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2405 00000100:00080000:2.0:1787827002.101206:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827002.101210:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827002.101221:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827002.101234:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011536896/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827002.101242:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827002.101245:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827002.101247:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827002.101250:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2405 00000100:00080000:2.0:1787827002.101254:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827002.101257:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827002.101261:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827002.101270:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011537024/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827002.101277:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827002.101280:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827002.101282:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827002.101285:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2405 00000100:00080000:2.0:1787827002.101288:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827002.101291:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827002.101295:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827002.101304:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011537152/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827002.101886:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011536768:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787827002.101966:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827002.102023:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827002/real 1787827002] req@ffff907d91148b40 x1874670011536768/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827002.102043:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011536768/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827002.102054:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827002.102058:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827002.102062:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011536768:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827002.102070:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011536896:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827002.102095:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827002.102119:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827002/real 1787827002] req@ffff907d91148f00 x1874670011536896/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827002.102132:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011536896/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827002.102140:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827002.102144:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827002.102146:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011536896:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827002.102153:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011537024:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827002.102175:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827002.102186:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827002/real 1787827002] req@ffff907d911492c0 x1874670011537024/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827002.102210:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011537024/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827002.102224:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827002.102228:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827002.102231:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011537024:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827002.102238:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011537152:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827002.102262:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827002.102276:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827002/real 1787827002] req@ffff907d9114a940 x1874670011537152/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827002.102288:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011537152/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827002.102296:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827002.102300:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827002.102304:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011537152:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827002.228756:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827002.228824:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827003.252915:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827003.253726:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827004.276772:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827004.277459:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787827005.300693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827005.300815:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827006.324760:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827006.324785:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827007.220787:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827007.220797:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827007.220802:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827007.220811:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2410 00000100:00080000:1.0:1787827007.220822:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827007.220829:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827007.220870:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011537280/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827007.220959:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011537280:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787827007.221023:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787827007.221057:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827007.221062:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827007.221064:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827007.221068:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2410 00000100:00080000:1.0:1787827007.221072:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827007.221076:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827007.221086:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827007.221099:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011537408/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827007.221176:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827007/real 1787827007] req@ffff907d908a3840 x1874670011537280/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827007.221198:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011537280/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827007.221226:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827007.221231:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827007.221235:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011537280:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827007.221262:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011537408:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827007.221287:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827007.221300:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827007/real 1787827007] req@ffff907d908a2580 x1874670011537408/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827007.221312:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011537408/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827007.221320:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827007.221323:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827007.221326:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011537408:192.168.203.104@tcp:38:kworker.0 00000100:00080000:1.0:1787827007.221389:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827007.221393:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827007.221395:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827007.221399:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2410 00000100:00080000:1.0:1787827007.221403:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827007.221408:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827007.221413:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827007.221425:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011537536/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827007.221434:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827007.221438:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827007.221439:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827007.221443:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2410 00000100:00080000:1.0:1787827007.221446:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827007.221450:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827007.221454:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827007.221464:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011537664/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827007.221560:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011537536:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827007.221585:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827007.221597:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827007/real 1787827007] req@ffff907d908a21c0 x1874670011537536/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827007.221643:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011537536/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827007.221652:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827007.221655:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827007.221658:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011537536:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827007.221664:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011537664:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827007.221683:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827007.221695:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827007/real 1787827007] req@ffff907d908a12c0 x1874670011537664/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827007.221704:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011537664/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827007.221711:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827007.221714:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827007.221717:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011537664:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827007.348767:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827007.348819:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827008.372770:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827008.375137:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787827009.396739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827009.396756:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827010.420709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827010.420739:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827011.444767:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827011.444773:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827012.340992:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827012.341001:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827012.341006:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827012.341015:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2416 00000100:00080000:2.0:1787827012.341028:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827012.341036:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827012.341065:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011537792/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827012.341094:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827012.341098:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827012.341101:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827012.341108:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2416 00000100:00080000:2.0:1787827012.341113:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827012.341119:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827012.341129:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827012.341138:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011537792:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827012.341150:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011537920/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827012.341162:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827012.341165:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827012.341167:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827012.341171:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2416 00000100:00080000:2.0:1787827012.341175:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827012.341179:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827012.341185:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827012.341198:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011538048/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827012.341211:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827012.341215:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827012.341218:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827012.341229:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2416 00000800:00000100:3.0:1787827012.341229:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787827012.341235:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827012.341241:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827012.341250:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827012.341265:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011538176/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827012.341279:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827012/real 1787827012] req@ffff907d9114ad00 x1874670011537792/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827012.341301:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011537792/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827012.341310:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827012.341313:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827012.341317:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011537792:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827012.341352:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011537920:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827012.341371:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827012.341392:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827012/real 1787827012] req@ffff907d91149a40 x1874670011537920/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827012.341404:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011537920/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827012.341413:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827012.341417:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827012.341420:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011537920:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827012.341502:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011538048:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827012.341525:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827012.341538:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827012/real 1787827012] req@ffff907d9114a580 x1874670011538048/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827012.341549:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011538048/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827012.341557:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827012.341561:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827012.341564:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011538048:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827012.341570:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011538176:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827012.341589:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827012.341600:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827012/real 1787827012] req@ffff907d9114b840 x1874670011538176/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827012.341781:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011538176/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827012.341788:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827012.341791:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827012.341793:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011538176:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827012.468756:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827012.468828:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827013.492704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827013.492725:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827014.516706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827014.516773:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827015.540673:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827015.540705:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827016.564772:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827016.565155:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827017.460749:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827017.460759:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827017.460763:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827017.460774:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2421 00000100:00080000:1.0:1787827017.460785:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827017.460792:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827017.460824:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011538304/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827017.460857:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827017.460861:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827017.460863:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827017.460867:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2421 00000100:00080000:1.0:1787827017.460871:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827017.460874:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827017.460883:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827017.460894:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011538432/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827017.460903:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827017.460906:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827017.460907:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827017.460911:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2421 00000100:00080000:1.0:1787827017.460913:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827017.460917:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827017.460921:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827017.460930:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011538560/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827017.460937:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827017.460940:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827017.460942:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827017.460945:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2421 00000100:00080000:1.0:1787827017.460947:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827017.460950:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827017.460954:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827017.460962:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011538688/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827017.461037:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011538304:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787827017.461095:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827017.461141:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827017/real 1787827017] req@ffff907d908a1680 x1874670011538304/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827017.461159:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011538304/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827017.461168:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827017.461171:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827017.461174:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011538304:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827017.461181:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011538432:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827017.461208:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827017.461222:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827017/real 1787827017] req@ffff907d908a0f00 x1874670011538432/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827017.461233:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011538432/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827017.461240:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827017.461243:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827017.461246:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011538432:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827017.461252:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011538560:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827017.461270:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827017.461280:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827017/real 1787827017] req@ffff907d908a12c0 x1874670011538560/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827017.461290:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011538560/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827017.461309:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827017.461312:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827017.461316:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011538560:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827017.461321:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011538688:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827017.461339:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827017.461349:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827017/real 1787827017] req@ffff907d908a21c0 x1874670011538688/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827017.461358:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011538688/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827017.461370:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827017.461373:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827017.461376:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011538688:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827017.588715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827017.588723:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827018.612714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827018.612752:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827019.636743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827019.636861:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827020.660719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827020.660889:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827021.684792:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827021.684862:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827022.580793:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827022.580809:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827022.580815:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827022.580831:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2426 00000100:00080000:2.0:1787827022.580845:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827022.580854:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827022.580906:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011538816/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827022.580939:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827022.580943:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827022.580944:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827022.580949:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2426 00000100:00080000:2.0:1787827022.580953:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827022.580957:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827022.580969:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827022.580983:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011538944/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827022.580994:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827022.580999:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1787827022.581003:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011538816:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827022.581013:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827022.581019:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2426 00000100:00080000:2.0:1787827022.581025:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827022.581029:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827022.581033:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827022.581046:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011539072/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827022.581056:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827022.581060:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827022.581062:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827022.581066:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2426 00000100:00080000:2.0:1787827022.581069:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827022.581072:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827022.581076:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827022.581085:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011539200/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827022.581113:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827022.581166:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827022/real 1787827022] req@ffff907d91149680 x1874670011538816/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827022.581187:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011538816/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827022.581199:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827022.581204:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827022.581209:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011538816:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827022.581240:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011538944:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827022.581276:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827022.581290:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827022/real 1787827022] req@ffff907d9114bc00 x1874670011538944/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827022.581302:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011538944/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827022.581311:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827022.581314:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827022.581318:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011538944:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827022.581323:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011539072:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827022.581346:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827022.581356:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827022/real 1787827022] req@ffff907d911483c0 x1874670011539072/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827022.581368:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011539072/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827022.581376:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827022.581380:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827022.581383:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011539072:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827022.581389:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011539200:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827022.581408:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827022.581418:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827022/real 1787827022] req@ffff907d9114b480 x1874670011539200/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827022.581429:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011539200/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827022.581437:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827022.581440:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827022.581444:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011539200:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827022.708731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827022.708764:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827023.732869:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827023.733349:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787827024.756704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827024.756728:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827025.780747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827025.780816:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827026.804745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827026.804753:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827027.700772:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827027.700786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827027.700791:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827027.700804:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2431 00000100:00080000:0.0:1787827027.700820:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827027.700829:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827027.700886:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011539328/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827027.700919:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827027.700923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827027.700924:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827027.700928:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2431 00000100:00080000:0.0:1787827027.700933:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827027.700936:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827027.700948:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827027.700965:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011539456/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827027.700973:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011539328:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787827027.700989:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827027.700992:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787827027.700994:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827027.701000:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2431 00000100:00080000:0.0:1787827027.701003:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827027.701007:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827027.701012:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827027.701024:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011539584/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827027.701034:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827027.701038:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827027.701040:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827027.701044:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2431 00000100:00080000:0.0:1787827027.701048:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787827027.701051:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787827027.701055:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827027.701062:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827027.701081:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011539712/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827027.701113:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827027/real 1787827027] req@ffff907d90846580 x1874670011539328/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827027.701131:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011539328/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827027.701141:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827027.701147:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827027.701150:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011539328:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827027.701171:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011539456:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827027.701187:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827027.701199:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827027/real 1787827027] req@ffff907d90847c00 x1874670011539456/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827027.701207:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011539456/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827027.701212:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827027.701214:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827027.701217:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011539456:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827027.701221:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011539584:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827027.701246:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827027.701254:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827027/real 1787827027] req@ffff907d908470c0 x1874670011539584/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827027.701261:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011539584/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827027.701266:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827027.701268:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827027.701270:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011539584:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827027.701274:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011539712:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827027.701283:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827027.701289:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827027/real 1787827027] req@ffff907d90845e00 x1874670011539712/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827027.701296:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011539712/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827027.701300:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827027.701302:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827027.701303:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011539712:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827027.828754:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827027.829098:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827028.852762:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827028.852848:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827029.876768:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827029.876838:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827030.900784:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827030.900825:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787827031.924725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827031.924766:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827032.820738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827032.820749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827032.820753:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827032.820764:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2436 00000100:00080000:0.0:1787827032.820774:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827032.820781:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827032.820817:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011539840/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827032.820850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827032.820854:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827032.820856:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827032.820859:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2436 00000100:00080000:0.0:1787827032.820863:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827032.820867:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827032.820878:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827032.820890:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011539968/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827032.820899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827032.820902:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787827032.820903:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827032.820907:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2436 00000100:00080000:0.0:1787827032.820910:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827032.820913:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827032.820917:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827032.820926:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011540096/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827032.820933:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827032.820935:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827032.820937:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827032.820939:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2436 00000100:00080000:0.0:1787827032.820941:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827032.820943:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827032.820946:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827032.820951:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011540224/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827032.821203:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011539840:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787827032.821280:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827032.821337:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827032/real 1787827032] req@ffff907d90844b40 x1874670011539840/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827032.821351:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011539840/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827032.821360:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827032.821363:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827032.821366:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011539840:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827032.821372:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011539968:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827032.821387:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827032.821395:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827032/real 1787827032] req@ffff907d908443c0 x1874670011539968/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827032.821411:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011539968/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827032.821418:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827032.821421:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827032.821423:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011539968:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827032.821427:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011540096:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827032.821446:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827032.821456:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827032/real 1787827032] req@ffff907d90844000 x1874670011540096/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827032.821474:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011540096/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827032.821481:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827032.821484:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827032.821487:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011540096:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827032.821493:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011540224:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827032.821510:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827032.821521:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827032/real 1787827032] req@ffff907d90844780 x1874670011540224/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827032.821532:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011540224/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827032.821540:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827032.821543:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827032.821547:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011540224:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827032.948768:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827032.948792:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827033.972739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827033.972810:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827034.996780:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827034.996843:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827036.020720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827036.020996:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827037.044864:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827037.044875:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827037.940720:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827037.940727:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827037.940731:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827037.940738:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2441 00000100:00080000:3.0:1787827037.940748:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827037.940755:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827037.940790:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011540352/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827037.940817:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827037.940821:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827037.940822:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827037.940826:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2441 00000100:00080000:3.0:1787827037.940830:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827037.940834:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827037.940844:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827037.940855:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011540352:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827037.940859:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011540480/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827037.940870:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827037.940879:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827037.940881:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827037.940885:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2441 00000100:00080000:3.0:1787827037.940889:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827037.940892:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827037.940897:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827037.940908:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011540608/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827037.940912:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787827037.940922:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827037.940925:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827037.940928:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827037.940933:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2441 00000100:00080000:3.0:1787827037.940937:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827037.940940:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827037.940946:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827037.940957:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827037/real 1787827037] req@ffff907da562f480 x1874670011540352/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827037.940959:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011540736/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827037.940998:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011540352/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827037.941007:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827037.941010:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827037.941013:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011540352:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827037.941031:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011540480:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827037.941048:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827037.941056:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827037/real 1787827037] req@ffff907da562c780 x1874670011540480/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827037.941063:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011540480/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827037.941068:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827037.941070:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827037.941072:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011540480:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827037.941075:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011540608:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827037.941086:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827037.941091:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827037/real 1787827037] req@ffff907da562ed00 x1874670011540608/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827037.941098:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011540608/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827037.941106:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827037.941108:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827037.941110:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011540608:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827037.941113:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011540736:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827037.941122:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827037.941126:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827037/real 1787827037] req@ffff907da562cf00 x1874670011540736/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827037.941132:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011540736/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827037.941137:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827037.941138:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827037.941140:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011540736:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827038.068722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827038.068768:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827039.092710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827039.092737:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827040.116713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827040.116798:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827041.140719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827041.140862:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827042.164760:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827042.164786:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827043.060754:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827043.060778:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827043.060783:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827043.060796:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2446 00000100:00080000:1.0:1787827043.060809:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827043.060818:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827043.060880:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011540864/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827043.060912:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827043.060917:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827043.060919:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827043.060924:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2446 00000100:00080000:1.0:1787827043.060928:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827043.060933:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827043.060951:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827043.060960:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011540864:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827043.060968:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011540992/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827043.060980:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827043.060983:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827043.060986:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827043.060990:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2446 00000100:00080000:1.0:1787827043.060994:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827043.060999:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827043.061004:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827043.061016:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011541120/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827043.061025:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827043.061029:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827043.061031:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827043.061034:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2446 00000800:00000100:0.0:1787827043.061041:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787827043.061046:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827043.061052:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827043.061058:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827043.061073:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011541248/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827043.061102:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827043/real 1787827043] req@ffff907d908a2580 x1874670011540864/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827043.061133:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011540864/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827043.061145:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827043.061150:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827043.061154:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011540864:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827043.061180:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011540992:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827043.061202:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827043.061211:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827043/real 1787827043] req@ffff907d908a3840 x1874670011540992/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827043.061219:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011540992/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827043.061224:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827043.061227:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827043.061229:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011540992:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827043.061233:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011541120:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827043.061245:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827043.061250:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827043/real 1787827043] req@ffff907d908a0000 x1874670011541120/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827043.061257:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011541120/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827043.061262:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827043.061263:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827043.061265:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011541120:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827043.061268:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011541248:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827043.061278:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827043.061283:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827043/real 1787827043] req@ffff907d908a2d00 x1874670011541248/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827043.061290:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011541248/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827043.061294:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827043.061296:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827043.061297:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011541248:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827043.188760:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827043.188797:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827044.212716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827044.212742:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827045.236761:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827045.236835:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827046.260749:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827046.260765:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827047.284819:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827047.284863:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:1.0:1787827048.180875:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827048.180895:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827048.180900:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827048.180910:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2451 00000100:00080000:1.0:1787827048.180919:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827048.180926:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827048.180958:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011541376/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827048.180983:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827048.180987:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827048.180988:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827048.180992:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2451 00000100:00080000:1.0:1787827048.180995:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827048.180998:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827048.181006:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827048.181016:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011541504/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827048.181026:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827048.181031:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787827048.181033:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011541376:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827048.181034:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827048.181043:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2451 00000100:00080000:1.0:1787827048.181047:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827048.181050:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827048.181055:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827048.181066:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011541632/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827048.181075:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827048.181078:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827048.181080:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827048.181083:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2451 00000100:00080000:1.0:1787827048.181106:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827048.181110:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827048.181114:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827048.181125:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011541760/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827048.181162:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011541504:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827048.181203:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011541632:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827048.181223:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011541760:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827048.308855:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827048.308861:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827049.332753:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827049.332885:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827050.356762:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827050.356798:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787827051.254127:0:2337:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:2.0:1787827051.254168:0:2337:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787827051.254196:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787827051.254232:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787827051.254241:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787827051.254248:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:0.0:1787827051.254322:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827048/real 1787827051] req@ffff907d908a1e00 x1874670011541504/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827051.254356:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011541504/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827051.254369:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827051.254375:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827051.254379:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011541504:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827051.254387:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827048/real 1787827051] req@ffff907d908a2d00 x1874670011541632/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827051.254398:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011541632/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827051.254406:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827051.254410:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827051.254414:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011541632:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827051.254420:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827048/real 1787827051] req@ffff907d908a0000 x1874670011541760/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827051.254431:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011541760/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827051.254438:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827051.254441:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827051.254444:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011541760:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827051.254451:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827048/real 1787827051] req@ffff907d908a2940 x1874670011541376/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827051.254462:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011541376/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827051.254469:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827051.254473:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827051.254476:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011541376:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827051.380736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827051.380805:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827052.404708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827052.404772:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827053.300783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827053.300793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827053.300797:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827053.300807:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2457 00000100:00080000:3.0:1787827053.300817:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827053.300825:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827053.300857:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011541888/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827053.300890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827053.300894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827053.300895:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827053.300900:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2457 00000100:00080000:3.0:1787827053.300903:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827053.300907:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827053.300917:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827053.300930:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011542016/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827053.300941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787827053.300944:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011541888:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827053.300946:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827053.300950:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827053.300956:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2457 00000100:00080000:3.0:1787827053.300959:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827053.300963:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827053.300968:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827053.300980:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011542144/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827053.300990:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827053.300993:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827053.300995:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827053.300999:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2457 00000100:00080000:3.0:1787827053.301003:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827053.301007:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827053.301011:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787827053.301014:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827053.301028:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011542272/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827053.301078:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827053/real 1787827053] req@ffff907da562d680 x1874670011541888/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827053.301097:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011541888/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827053.301108:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827053.301112:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827053.301116:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011541888:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827053.301138:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011542016:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827053.301161:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827053.301174:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827053/real 1787827053] req@ffff907da562cf00 x1874670011542016/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827053.301186:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011542016/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827053.301195:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827053.301198:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827053.301201:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011542016:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827053.301222:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011542144:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827053.301246:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827053.301258:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827053/real 1787827053] req@ffff907da562ed00 x1874670011542144/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827053.301270:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011542144/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827053.301278:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827053.301281:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827053.301284:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011542144:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827053.301290:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011542272:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827053.301310:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827053.301321:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827053/real 1787827053] req@ffff907da562c780 x1874670011542272/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827053.301332:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011542272/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827053.301340:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827053.301343:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827053.301347:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011542272:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827053.428762:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827053.428766:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827054.452716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827054.452776:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827055.476736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827055.476991:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827056.500757:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827056.500785:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827057.524747:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827057.524832:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:1.0:1787827058.420748:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827058.420758:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827058.420762:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827058.420772:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2462 00000100:00080000:1.0:1787827058.420783:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827058.420797:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827058.420831:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011542400/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827058.420860:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827058.420863:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827058.420865:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827058.420880:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2462 00000100:00080000:1.0:1787827058.420884:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827058.420889:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827058.420898:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827058.420909:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011542400:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827058.420913:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011542528/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827058.420925:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827058.420928:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827058.420930:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827058.420934:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2462 00000100:00080000:1.0:1787827058.420937:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827058.420941:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827058.420945:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827058.420958:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011542656/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827058.420965:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787827058.420972:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827058.420976:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827058.420979:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827058.420984:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2462 00000100:00080000:1.0:1787827058.420987:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827058.420991:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827058.420996:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827058.421000:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827058/real 1787827058] req@ffff907d908a3840 x1874670011542400/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827058.421008:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011542784/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827058.421041:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011542400/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827058.421053:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827058.421058:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827058.421061:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011542400:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827058.421083:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011542528:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827058.421107:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827058.421119:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827058/real 1787827058] req@ffff907d908a2580 x1874670011542528/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827058.421131:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011542528/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827058.421139:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827058.421142:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827058.421145:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011542528:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827058.421150:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011542656:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827058.421170:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827058.421181:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827058/real 1787827058] req@ffff907d908a21c0 x1874670011542656/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827058.421188:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011542656/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827058.421197:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827058.421199:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827058.421201:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011542656:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827058.421204:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011542784:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827058.421216:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827058.421222:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827058/real 1787827058] req@ffff907d908a12c0 x1874670011542784/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827058.421228:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011542784/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827058.421234:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827058.421236:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827058.421240:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011542784:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787827058.548819:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827058.548902:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787827059.572701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827059.572854:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827060.596754:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827060.596845:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827061.621081:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827061.621335:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787827062.644756:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827062.644895:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827063.540718:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827063.540727:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827063.540732:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827063.540742:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2467 00000100:00080000:1.0:1787827063.540753:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827063.540761:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827063.540797:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011542912/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827063.540830:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827063.540834:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827063.540836:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827063.540841:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2467 00000100:00080000:1.0:1787827063.540844:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827063.540848:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827063.540857:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827063.540870:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011543040/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827063.540880:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827063.540882:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827063.540883:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827063.540886:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2467 00000100:00080000:1.0:1787827063.540888:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827063.540891:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827063.540894:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827063.540902:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011543168/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827063.540908:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827063.540910:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827063.540911:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827063.540913:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2467 00000100:00080000:1.0:1787827063.540914:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827063.540916:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827063.540918:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827063.540923:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011543296/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827063.541044:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011542912:192.168.203.104@tcp:250:kworker.0 00000800:00000100:0.0:1787827063.541103:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827063.541149:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827063/real 1787827063] req@ffff907d908a0f00 x1874670011542912/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827063.541167:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011542912/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827063.541177:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827063.541181:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827063.541186:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011542912:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827063.541193:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011543040:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827063.541214:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827063.541226:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827063/real 1787827063] req@ffff907d908a1680 x1874670011543040/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827063.541238:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011543040/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827063.541245:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827063.541248:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827063.541251:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011543040:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827063.541257:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011543168:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827063.541275:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827063.541284:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827063/real 1787827063] req@ffff907d908a12c0 x1874670011543168/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827063.541295:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011543168/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827063.541302:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827063.541305:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827063.541308:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011543168:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827063.541314:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011543296:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827063.541331:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827063.541341:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827063/real 1787827063] req@ffff907d908a21c0 x1874670011543296/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827063.541352:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011543296/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827063.541359:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827063.541362:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827063.541365:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011543296:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827063.669100:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827063.669333:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827064.692748:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827064.692840:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787827065.716707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827065.716815:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827066.740733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827066.740905:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827067.764745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827067.764793:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827068.660718:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827068.660728:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827068.660732:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827068.660743:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2472 00000100:00080000:1.0:1787827068.660755:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827068.660763:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827068.660799:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011543424/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827068.660832:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827068.660835:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827068.660837:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827068.660841:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2472 00000100:00080000:1.0:1787827068.660845:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827068.660849:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827068.660859:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827068.660872:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011543552/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827068.660882:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827068.660885:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827068.660887:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827068.660891:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2472 00000100:00080000:1.0:1787827068.660894:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827068.660897:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827068.660902:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827068.660914:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011543680/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827068.660924:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827068.660927:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827068.660928:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827068.660931:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2472 00000100:00080000:1.0:1787827068.660934:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827068.660938:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827068.660952:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827068.660963:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011543808/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827068.661069:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011543424:192.168.203.104@tcp:250:kworker.0 00000800:00000100:0.0:1787827068.661141:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827068.661192:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827068/real 1787827068] req@ffff907d908a2580 x1874670011543424/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827068.661206:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011543424/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827068.661216:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827068.661220:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827068.661224:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011543424:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827068.661230:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011543552:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827068.661251:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827068.661265:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827068/real 1787827068] req@ffff907d908a3840 x1874670011543552/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827068.661283:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011543552/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827068.661291:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827068.661294:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827068.661297:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011543552:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827068.661303:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011543680:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827068.661324:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827068.661335:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827068/real 1787827068] req@ffff907d908a2940 x1874670011543680/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827068.661346:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011543680/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827068.661353:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827068.661356:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827068.661359:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011543680:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827068.661364:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011543808:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827068.661383:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827068.661394:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827068/real 1787827068] req@ffff907d908a0000 x1874670011543808/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827068.661414:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011543808/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827068.661430:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827068.661434:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827068.661436:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011543808:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827068.788768:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827068.789005:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827069.812828:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827069.812899:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827070.836753:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827070.836913:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787827071.860690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827071.860710:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827072.884711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827072.884759:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827073.780770:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827073.780783:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827073.780789:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827073.780802:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2477 00000100:00080000:1.0:1787827073.780817:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827073.780826:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827073.780873:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011543936/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827073.780905:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827073.780926:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827073.780928:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827073.780933:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2477 00000100:00080000:1.0:1787827073.780937:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827073.780942:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827073.780954:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827073.780964:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011543936:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827073.780970:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011544064/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827073.780983:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827073.780987:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827073.780989:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827073.780993:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2477 00000100:00080000:1.0:1787827073.780996:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827073.781000:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827073.781005:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827073.781020:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011544192/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827073.781030:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827073.781034:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827073.781036:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827073.781040:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2477 00000100:00080000:1.0:1787827073.781044:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827073.781048:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827073.781053:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787827073.781058:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827073.781069:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011544320/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827073.781114:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827073/real 1787827073] req@ffff907d908a2d00 x1874670011543936/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827073.781128:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011543936/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827073.781137:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827073.781140:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827073.781144:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011543936:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827073.781165:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011544064:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827073.781181:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827073.781189:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827073/real 1787827073] req@ffff907d908a1e00 x1874670011544064/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827073.781196:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011544064/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827073.781204:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827073.781207:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827073.781210:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011544064:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827073.781216:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011544192:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827073.781240:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827073.781252:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827073/real 1787827073] req@ffff907d908a0000 x1874670011544192/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827073.781263:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011544192/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827073.781270:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827073.781273:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827073.781276:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011544192:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827073.781281:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011544320:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827073.781300:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827073.781311:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827073/real 1787827073] req@ffff907d908a2940 x1874670011544320/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827073.781321:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011544320/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827073.781328:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827073.781331:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827073.781335:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011544320:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827073.908730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827073.908846:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827074.932819:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827074.932921:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827075.956767:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827075.956809:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827076.980824:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827076.980844:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787827078.004752:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827078.004752:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827078.900707:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827078.900717:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827078.900722:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827078.900733:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2482 00000100:00080000:2.0:1787827078.900744:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827078.900753:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827078.900791:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011544448/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827078.900819:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827078.900823:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827078.900824:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827078.900827:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2482 00000100:00080000:2.0:1787827078.900830:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827078.900833:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827078.900842:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827078.900853:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011544576/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827078.900862:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011544448:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827078.900863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827078.900883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827078.900887:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827078.900891:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2482 00000100:00080000:2.0:1787827078.900895:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827078.900899:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827078.900904:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827078.900917:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011544704/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827078.900926:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827078.900929:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827078.900931:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827078.900934:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2482 00000100:00080000:2.0:1787827078.900938:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827078.900942:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827078.900947:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787827078.900951:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827078.900961:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011544832/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827078.900997:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827078/real 1787827078] req@ffff907d9114b480 x1874670011544448/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827078.901014:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011544448/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827078.901024:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827078.901028:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827078.901031:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011544448:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827078.901052:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011544576:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827078.901078:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827078.901089:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827078/real 1787827078] req@ffff907d911483c0 x1874670011544576/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827078.901100:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011544576/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827078.901109:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827078.901112:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827078.901115:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011544576:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827078.901121:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011544704:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827078.901140:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827078.901150:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827078/real 1787827078] req@ffff907d9114bc00 x1874670011544704/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827078.901179:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011544704/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827078.901187:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827078.901190:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827078.901193:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011544704:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827078.901199:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011544832:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827078.901218:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827078.901228:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827078/real 1787827078] req@ffff907d91149680 x1874670011544832/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827078.901239:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011544832/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827078.901246:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827078.901249:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827078.901252:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011544832:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827079.028725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827079.028753:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827080.052722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827080.052822:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827081.076779:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827081.076802:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827082.100775:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827082.101044:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827083.124733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827083.124755:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827084.020727:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827084.020738:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827084.020743:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827084.020753:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2487 00000100:00080000:2.0:1787827084.020764:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827084.020772:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827084.020805:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011544960/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827084.020845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827084.020849:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827084.020851:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827084.020857:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2487 00000100:00080000:2.0:1787827084.020861:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827084.020864:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827084.020874:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827084.020888:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011545088/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827084.020908:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011544960:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827084.020917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827084.020921:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827084.020923:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827084.020928:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2487 00000100:00080000:2.0:1787827084.020932:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827084.020936:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827084.020942:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827084.020958:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011545216/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827084.020967:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827084.020971:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827084.020974:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787827084.020978:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787827084.020982:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2487 00000100:00080000:2.0:1787827084.020987:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827084.020992:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827084.021001:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827084.021018:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011545344/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827084.021020:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827084/real 1787827084] req@ffff907d9114b840 x1874670011544960/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827084.021060:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011544960/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827084.021077:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827084.021084:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827084.021093:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011544960:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827084.021124:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011545088:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827084.021165:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827084.021187:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827084/real 1787827084] req@ffff907d9114a580 x1874670011545088/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827084.021208:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011545088/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827084.021215:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827084.021218:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827084.021220:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011545088:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827084.021227:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011545216:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827084.021267:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827084.021283:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827084/real 1787827084] req@ffff907d91149a40 x1874670011545216/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827084.021297:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011545216/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827084.021305:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827084.021309:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827084.021312:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011545216:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827084.021320:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011545344:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827084.021342:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827084.021354:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827084/real 1787827084] req@ffff907d9114ad00 x1874670011545344/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827084.021370:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011545344/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827084.021376:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827084.021379:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827084.021381:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011545344:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827084.148754:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827084.148813:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827085.172758:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827085.172850:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827086.196776:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827086.196799:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827087.220785:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827087.221021:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827088.244740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827088.244885:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827089.140775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827089.140787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827089.140793:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827089.140816:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2492 00000100:00080000:3.0:1787827089.140830:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827089.140838:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827089.140893:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011545472/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827089.140923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827089.140927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827089.140930:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827089.140934:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2492 00000100:00080000:3.0:1787827089.140938:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827089.140941:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827089.140953:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827089.140966:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011545600/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827089.140971:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011545472:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827089.140984:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827089.140987:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827089.140989:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827089.140994:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2492 00000100:00080000:3.0:1787827089.141005:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827089.141010:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827089.141016:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827089.141028:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011545728/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827089.141037:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787827089.141038:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787827089.141044:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827089.141046:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827089.141051:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2492 00000100:00080000:3.0:1787827089.141056:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827089.141060:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827089.141065:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827089.141079:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011545856/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827089.141102:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827089/real 1787827089] req@ffff907da562f480 x1874670011545472/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827089.141120:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011545472/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827089.141128:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827089.141131:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827089.141135:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011545472:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827089.141157:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011545600:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827089.141174:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827089.141183:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827089/real 1787827089] req@ffff907da562c000 x1874670011545600/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827089.141191:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011545600/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827089.141199:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827089.141209:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827089.141213:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011545600:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827089.141220:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011545728:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827089.141241:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827089.141252:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827089/real 1787827089] req@ffff907da562e580 x1874670011545728/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827089.141260:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011545728/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827089.141265:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827089.141267:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827089.141270:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011545728:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827089.141273:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011545856:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827089.141285:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827089.141290:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827089/real 1787827089] req@ffff907da562d2c0 x1874670011545856/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827089.141297:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011545856/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827089.141305:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827089.141309:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827089.141311:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011545856:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827089.268764:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827089.268921:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827090.292725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827090.292808:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827091.316752:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827091.316938:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827092.340954:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827092.342540:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787827093.364721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827093.364848:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827094.261192:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827094.261205:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827094.261210:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827094.261223:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2498 00000100:00080000:3.0:1787827094.261237:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827094.261246:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827094.261292:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011545984/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827094.261325:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827094.261329:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827094.261330:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827094.261334:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2498 00000100:00080000:3.0:1787827094.261337:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827094.261341:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827094.261353:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827094.261365:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011546112/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827094.261375:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827094.261379:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827094.261381:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827094.261386:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2498 00000100:00080000:3.0:1787827094.261389:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827094.261392:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827094.261398:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827094.261403:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011545984:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827094.261410:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011546240/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827094.261420:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827094.261424:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827094.261426:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827094.261430:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2498 00000100:00080000:3.0:1787827094.261433:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827094.261437:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827094.261441:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827094.261450:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011546368/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827094.261553:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827094.261605:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827094/real 1787827094] req@ffff907da562c3c0 x1874670011545984/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827094.261650:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011545984/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827094.261662:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827094.261667:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827094.261671:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011545984:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827094.261679:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011546112:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827094.261706:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827094.261721:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827094/real 1787827094] req@ffff907da562f0c0 x1874670011546112/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827094.261734:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011546112/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827094.261742:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827094.261746:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827094.261750:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011546112:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827094.261793:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011546240:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827094.261817:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827094.261830:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827094/real 1787827094] req@ffff907da562fc00 x1874670011546240/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827094.261842:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011546240/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827094.261851:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827094.261853:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827094.261856:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011546240:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827094.261862:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011546368:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827094.261880:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827094.261891:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827094/real 1787827094] req@ffff907da562e940 x1874670011546368/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827094.261902:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011546368/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827094.261909:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827094.261913:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827094.261916:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011546368:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787827094.388872:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827094.388929:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787827095.412828:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827095.412908:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827096.436796:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827096.437033:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827097.460820:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827097.460979:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827098.484714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827098.484797:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827099.380670:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827099.380680:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827099.380684:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827099.380694:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2503 00000100:00080000:3.0:1787827099.380705:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827099.380711:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827099.380754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011546496/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827099.380773:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827099.380775:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827099.380777:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827099.380779:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2503 00000100:00080000:3.0:1787827099.380782:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827099.380784:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827099.380792:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827099.380800:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011546624/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827099.380807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827099.380810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827099.380811:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827099.380816:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2503 00000100:00100000:0.0:1787827099.380820:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011546496:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827099.380822:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827099.380828:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827099.380831:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827099.380838:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011546752/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827099.380849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827099.380851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827099.380852:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827099.380854:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2503 00000100:00080000:3.0:1787827099.380857:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827099.380859:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827099.380862:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827099.380869:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011546880/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827099.380893:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827099.380928:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827099/real 1787827099] req@ffff907da562e1c0 x1874670011546496/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827099.380941:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011546496/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827099.380948:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827099.380950:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827099.380953:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011546496:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827099.380969:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011546624:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827099.380984:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827099.380992:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827099/real 1787827099] req@ffff907da562f840 x1874670011546624/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827099.380999:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011546624/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827099.381004:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827099.381005:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827099.381007:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011546624:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827099.381010:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011546752:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827099.381020:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827099.381028:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827099/real 1787827099] req@ffff907da562de00 x1874670011546752/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827099.381038:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011546752/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827099.381051:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827099.381054:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827099.381057:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011546752:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827099.381062:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011546880:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827099.381079:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827099.381094:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827099/real 1787827099] req@ffff907da562cb40 x1874670011546880/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827099.381101:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011546880/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827099.381106:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827099.381108:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827099.381109:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011546880:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827099.508718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827099.508828:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827100.532720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827100.532761:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827101.556753:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827101.556822:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827102.580763:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827102.580818:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827103.604734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827103.604776:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827104.500794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827104.500811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827104.500817:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827104.500829:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2508 00000100:00080000:3.0:1787827104.500844:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827104.500853:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827104.500903:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011547008/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827104.500941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827104.500945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827104.500948:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827104.500953:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2508 00000100:00080000:3.0:1787827104.500956:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827104.500960:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827104.500972:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827104.500989:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011547136/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827104.501001:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827104.501004:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827104.501006:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827104.501010:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2508 00000100:00080000:3.0:1787827104.501014:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827104.501019:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827104.501025:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827104.501038:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011547008:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827104.501040:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011547264/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827104.501066:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827104.501070:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827104.501071:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827104.501076:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2508 00000100:00080000:3.0:1787827104.501080:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827104.501084:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827104.501096:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827104.501113:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011547392/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827104.501131:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827104.501174:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827104/real 1787827104] req@ffff907da562da40 x1874670011547008/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827104.501187:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011547008/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827104.501194:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827104.501198:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827104.501201:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011547008:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827104.501207:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011547136:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827104.501222:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827104.501231:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827104/real 1787827104] req@ffff907da562cb40 x1874670011547136/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827104.501238:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011547136/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827104.501243:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827104.501245:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827104.501246:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011547136:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827104.501265:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011547264:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827104.501284:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827104.501291:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827104/real 1787827104] req@ffff907da562de00 x1874670011547264/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827104.501300:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011547264/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827104.501308:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827104.501312:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827104.501315:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011547264:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827104.501324:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011547392:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827104.501340:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827104.501349:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827104/real 1787827104] req@ffff907da562f840 x1874670011547392/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827104.501359:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011547392/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827104.501366:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827104.501369:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827104.501372:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011547392:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827104.628746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827104.628833:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827105.652751:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827105.652814:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827106.676752:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827106.676811:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827107.700718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827107.700738:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827108.724758:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827108.724841:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827109.620795:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827109.620805:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827109.620810:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827109.620819:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2513 00000100:00080000:2.0:1787827109.620831:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827109.620838:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827109.620874:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011547520/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827109.620910:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827109.620914:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827109.620916:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827109.620920:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2513 00000100:00080000:2.0:1787827109.620924:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827109.620928:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827109.620937:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827109.620962:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011547648/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827109.620973:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827109.620976:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827109.620978:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827109.620983:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2513 00000100:00100000:0.0:1787827109.620984:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011547520:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827109.620994:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827109.620999:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827109.621005:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827109.621017:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011547776/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827109.621029:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827109.621031:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827109.621032:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827109.621034:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2513 00000100:00080000:2.0:1787827109.621037:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827109.621040:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827109.621045:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827109.621057:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011547904/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827109.621058:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827109.621104:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827109/real 1787827109] req@ffff907d9114a940 x1874670011547520/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827109.621131:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011547520/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827109.621142:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827109.621145:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827109.621150:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011547520:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827109.621157:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011547648:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827109.621179:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827109.621190:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827109/real 1787827109] req@ffff907d911492c0 x1874670011547648/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827109.621215:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011547648/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827109.621225:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827109.621228:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827109.621238:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011547648:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827109.621260:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011547776:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827109.621281:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827109.621292:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827109/real 1787827109] req@ffff907d91148f00 x1874670011547776/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827109.621303:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011547776/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827109.621312:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827109.621315:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827109.621318:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011547776:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827109.621323:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011547904:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827109.621341:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827109.621351:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827109/real 1787827109] req@ffff907d91148b40 x1874670011547904/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827109.621362:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011547904/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827109.621369:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827109.621372:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827109.621376:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011547904:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787827109.750058:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827109.751796:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787827110.772699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827110.772808:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827111.796737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827111.796828:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827112.820730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827112.820819:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827113.844787:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827113.844846:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827114.740729:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827114.740739:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827114.740743:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827114.740752:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2518 00000100:00080000:1.0:1787827114.740762:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827114.740769:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827114.740802:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011548032/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827114.740828:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827114.740832:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827114.740834:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827114.740837:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2518 00000100:00080000:1.0:1787827114.740840:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827114.740844:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827114.740854:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827114.740867:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011548160/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827114.740871:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011548032:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827114.740886:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827114.740889:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827114.740891:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827114.740895:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2518 00000100:00080000:1.0:1787827114.740898:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827114.740902:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827114.740907:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827114.740918:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011548288/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827114.740927:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827114.740930:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827114.740931:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827114.740934:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2518 00000100:00080000:1.0:1787827114.740937:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827114.740941:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827114.740945:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827114.740953:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011548416/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827114.740979:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011548160:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827114.741015:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011548288:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827114.741036:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011548416:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827114.868761:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827114.868907:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827115.892727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827115.892851:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827116.916729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827116.918445:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787827117.812918:0:2338:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:2.0:1787827117.812959:0:2338:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787827117.812974:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787827117.813015:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787827117.813023:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787827117.813029:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:0.0:1787827117.813895:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827114/real 1787827117] req@ffff907d908a2580 x1874670011548160/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827117.813918:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011548160/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827117.813930:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827117.813935:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827117.813939:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011548160:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827117.813946:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827114/real 1787827117] req@ffff907d908a21c0 x1874670011548288/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827117.813956:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011548288/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827117.813963:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827117.813967:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827117.813970:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011548288:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827117.813978:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827114/real 1787827117] req@ffff907d908a12c0 x1874670011548416/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827117.813989:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011548416/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827117.813996:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827117.813999:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827117.814002:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011548416:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827117.814008:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827114/real 1787827117] req@ffff907d908a3840 x1874670011548032/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827117.814019:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011548032/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827117.814026:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827117.814030:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827117.814033:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011548032:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827117.940710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827117.940842:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827118.964732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827118.964804:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827119.860811:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827119.860825:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827119.860831:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827119.860845:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2523 00000100:00080000:1.0:1787827119.860860:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827119.860870:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827119.860922:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011548544/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827119.860962:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827119.860967:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827119.860969:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827119.860973:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2523 00000100:00080000:1.0:1787827119.860977:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827119.860991:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827119.861004:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827119.861017:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011548672/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827119.861028:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827119.861031:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827119.861033:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827119.861037:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2523 00000100:00080000:1.0:1787827119.861041:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827119.861045:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827119.861056:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827119.861069:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011548800/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827119.861085:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827119.861089:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827119.861091:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827119.861094:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2523 00000100:00080000:1.0:1787827119.861098:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827119.861101:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827119.861117:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827119.861130:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011548928/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827119.863106:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011548544:192.168.203.104@tcp:250:kworker.0 00000800:00000100:0.0:1787827119.863209:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827119.863271:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827119/real 1787827119] req@ffff907d908a1680 x1874670011548544/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827119.863293:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011548544/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827119.863307:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827119.863313:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827119.863318:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011548544:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827119.863327:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011548672:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827119.863354:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827119.863368:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827119/real 1787827119] req@ffff907d908a0f00 x1874670011548672/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827119.863394:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011548672/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827119.863404:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827119.863407:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827119.863411:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011548672:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827119.863419:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011548800:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827119.863444:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827119.863457:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827119/real 1787827119] req@ffff907d908a3840 x1874670011548800/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827119.863469:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011548800/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827119.863477:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827119.863481:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827119.863484:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011548800:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827119.863492:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011548928:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827119.863515:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827119.863526:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827119/real 1787827119] req@ffff907d908a12c0 x1874670011548928/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827119.863538:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011548928/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827119.863546:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827119.863549:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827119.863552:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011548928:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827119.988735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827119.988780:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827121.012773:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827121.013015:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787827122.036919:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827122.038767:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787827123.060752:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827123.060861:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827124.084779:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827124.084947:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:2.0:1787827124.980740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827124.980751:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827124.980755:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827124.980765:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2528 00000100:00080000:2.0:1787827124.980776:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827124.980796:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827124.980831:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011549056/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827124.980860:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827124.980864:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827124.980866:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827124.980870:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2528 00000100:00080000:2.0:1787827124.980873:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827124.980878:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827124.980888:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827124.980900:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011549184/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827124.980909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827124.980919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827124.980921:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827124.980925:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2528 00000100:00080000:2.0:1787827124.980930:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787827124.980931:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011549056:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827124.980939:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827124.980945:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827124.980956:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011549312/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827124.980966:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827124.980970:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827124.980972:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827124.980975:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2528 00000100:00080000:2.0:1787827124.980978:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827124.980982:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827124.980987:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827124.980998:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011549440/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827124.981004:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827124.981050:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827124/real 1787827124] req@ffff907d91148b40 x1874670011549056/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827124.981063:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011549056/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827124.981072:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827124.981075:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827124.981079:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011549056:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827124.981086:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011549184:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827124.981108:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827124.981118:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827124/real 1787827124] req@ffff907d91148f00 x1874670011549184/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827124.981146:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011549184/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827124.981154:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827124.981157:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827124.981161:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011549184:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827124.981181:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011549312:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827124.981201:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827124.981212:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827124/real 1787827124] req@ffff907d911492c0 x1874670011549312/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827124.981224:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011549312/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827124.981232:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827124.981234:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827124.981237:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011549312:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827124.981241:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011549440:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827124.981254:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827124.981260:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827124/real 1787827124] req@ffff907d9114a940 x1874670011549440/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827124.981267:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011549440/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827124.981275:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827124.981277:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827124.981279:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011549440:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827125.109168:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827125.111640:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827126.132747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827126.132784:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827127.156708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827127.156765:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827128.180718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827128.180795:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827129.204723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827129.204776:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827130.100739:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827130.100746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827130.100750:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827130.100758:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2533 00000100:00080000:2.0:1787827130.100768:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827130.100773:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827130.100801:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011549568/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827130.100824:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827130.100826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827130.100827:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827130.100830:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2533 00000100:00080000:2.0:1787827130.100831:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827130.100833:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827130.100841:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827130.100850:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011549696/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827130.100856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827130.100859:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827130.100860:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827130.100862:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2533 00000100:00080000:2.0:1787827130.100864:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827130.100866:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827130.100868:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827130.100876:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011549824/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827130.100882:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011549568:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827130.100882:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827130.100887:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827130.100890:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827130.100901:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2533 00000100:00080000:2.0:1787827130.100908:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827130.100913:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827130.100918:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827130.100931:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011549952/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827130.100954:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827130.100994:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827130/real 1787827130] req@ffff907d9114ad00 x1874670011549568/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827130.101010:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011549568/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827130.101020:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827130.101024:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827130.101028:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011549568:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827130.101034:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011549696:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827130.101057:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827130.101069:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827130/real 1787827130] req@ffff907d91149a40 x1874670011549696/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827130.101079:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011549696/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827130.101087:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827130.101090:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827130.101099:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011549696:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827130.101121:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011549824:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827130.101141:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827130.101152:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827130/real 1787827130] req@ffff907d9114a580 x1874670011549824/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827130.101163:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011549824/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827130.101171:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827130.101174:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827130.101177:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011549824:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827130.101181:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011549952:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827130.101195:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827130.101203:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827130/real 1787827130] req@ffff907d9114b840 x1874670011549952/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827130.101213:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011549952/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827130.101220:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827130.101223:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827130.101226:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011549952:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827130.228744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827130.228824:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827131.252767:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827131.252842:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827132.276702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827132.276776:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827133.300766:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827133.300836:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787827134.324735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827134.324814:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827135.220719:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827135.220733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827135.220738:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827135.220749:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2538 00000100:00080000:2.0:1787827135.220760:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827135.220766:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827135.220803:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011550080/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827135.220829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827135.220832:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827135.220834:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827135.220838:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2538 00000100:00080000:2.0:1787827135.220841:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827135.220844:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827135.220854:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827135.220865:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011550208/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827135.220873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827135.220876:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827135.220877:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787827135.220884:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011550080:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827135.220884:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2538 00000100:00080000:2.0:1787827135.220890:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827135.220895:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827135.220900:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827135.220912:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011550336/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827135.220921:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827135.220925:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827135.220927:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827135.220930:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2538 00000100:00080000:2.0:1787827135.220935:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827135.220938:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827135.220943:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827135.220954:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011550464/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827135.220974:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827135.221016:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827135/real 1787827135] req@ffff907d91149680 x1874670011550080/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827135.221034:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011550080/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827135.221044:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827135.221048:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827135.221052:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011550080:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827135.221073:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011550208:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827135.221106:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827135.221119:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827135/real 1787827135] req@ffff907d9114bc00 x1874670011550208/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827135.221131:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011550208/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827135.221140:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827135.221143:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827135.221146:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011550208:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827135.221153:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011550336:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827135.221172:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827135.221199:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827135/real 1787827135] req@ffff907d911483c0 x1874670011550336/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827135.221210:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011550336/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827135.221228:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827135.221231:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827135.221234:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011550336:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827135.221240:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011550464:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827135.221259:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827135.221269:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827135/real 1787827135] req@ffff907d9114b480 x1874670011550464/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827135.221288:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011550464/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827135.221295:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827135.221298:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827135.221301:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011550464:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827135.348739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827135.348801:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827136.372724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827136.372763:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827137.396697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827137.396745:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827138.420732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827138.420788:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827139.444754:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827139.444825:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827140.341134:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827140.341144:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827140.341148:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827140.341158:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2544 00000100:00080000:0.0:1787827140.341169:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827140.341177:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827140.341211:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011550592/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827140.341239:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827140.341243:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827140.341245:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827140.341249:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2544 00000100:00080000:0.0:1787827140.341253:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827140.341257:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827140.341267:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827140.341280:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011550720/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827140.341291:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787827140.341291:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011550592:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787827140.341300:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787827140.341303:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827140.341307:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2544 00000100:00080000:0.0:1787827140.341310:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827140.341314:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827140.341319:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827140.341331:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011550848/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827140.341339:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827140.341342:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827140.341344:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827140.341347:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2544 00000100:00080000:0.0:1787827140.341351:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827140.341354:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827140.341360:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787827140.341364:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827140.341375:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011550976/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827140.341405:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827140/real 1787827140] req@ffff907d90845680 x1874670011550592/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827140.341422:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011550592/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827140.341432:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827140.341435:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827140.341439:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011550592:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827140.341476:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011550720:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827140.341499:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827140.341511:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827140/real 1787827140] req@ffff907d90846d00 x1874670011550720/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827140.341521:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011550720/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827140.341540:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827140.341544:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827140.341547:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011550720:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827140.341553:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011550848:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827140.341573:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827140.341583:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827140/real 1787827140] req@ffff907d908452c0 x1874670011550848/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827140.341595:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011550848/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827140.341603:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827140.341633:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827140.341637:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011550848:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827140.341642:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011550976:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827140.341660:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827140.341671:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827140/real 1787827140] req@ffff907d90845a40 x1874670011550976/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827140.341682:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011550976/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827140.341690:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827140.341693:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827140.341695:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011550976:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827140.468722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827140.468757:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827141.492705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827141.492768:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827142.516771:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827142.517554:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787827143.541228:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787827143.541283:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787827144.564761:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827144.564785:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827145.460762:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827145.460772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827145.460776:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827145.460790:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2549 00000100:00080000:2.0:1787827145.460801:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827145.460808:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827145.460847:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011551104/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827145.460883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827145.460887:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827145.460889:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827145.460893:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2549 00000100:00080000:2.0:1787827145.460896:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827145.460900:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827145.460910:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827145.460923:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011551232/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827145.460932:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827145.460934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827145.460936:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827145.460939:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2549 00000100:00080000:2.0:1787827145.460942:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827145.460945:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827145.460950:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827145.460959:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011551360/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827145.460968:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827145.460971:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827145.460973:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827145.460976:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2549 00000100:00080000:2.0:1787827145.460979:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827145.460982:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827145.460986:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827145.460996:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011551488/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827145.461071:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011551104:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787827145.461131:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827145.461166:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827145/real 1787827145] req@ffff907d9114b480 x1874670011551104/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827145.461188:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011551104/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827145.461203:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827145.461207:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827145.461211:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011551104:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827145.461218:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011551232:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827145.461235:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827145.461243:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827145/real 1787827145] req@ffff907d911483c0 x1874670011551232/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827145.461250:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011551232/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827145.461255:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827145.461257:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827145.461259:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011551232:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827145.461263:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011551360:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827145.461273:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827145.461278:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827145/real 1787827145] req@ffff907d9114bc00 x1874670011551360/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827145.461285:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011551360/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827145.461289:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827145.461291:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827145.461292:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011551360:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827145.461295:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011551488:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827145.461304:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827145.461309:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827145/real 1787827145] req@ffff907d91149680 x1874670011551488/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827145.461315:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011551488/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827145.461319:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827145.461321:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827145.461322:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011551488:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827145.588726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827145.588875:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827146.612712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827146.612913:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827147.636782:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827147.636820:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827148.660739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827148.660775:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827149.684710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827149.684771:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827150.580736:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827150.580746:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827150.580749:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827150.580757:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2554 00000100:00080000:1.0:1787827150.580766:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827150.580778:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827150.580813:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011551616/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827150.580850:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827150.580854:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827150.580856:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827150.580860:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2554 00000100:00080000:1.0:1787827150.580864:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827150.580869:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827150.580877:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827150.580889:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011551744/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827150.580896:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827150.580897:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827150.580898:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827150.580900:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2554 00000100:00080000:1.0:1787827150.580902:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827150.580904:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827150.580907:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827150.580913:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011551872/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827150.580918:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827150.580920:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827150.580922:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827150.580924:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2554 00000100:00080000:1.0:1787827150.580927:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827150.580930:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827150.580934:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827150.580949:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011552000/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827150.581035:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011551616:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787827150.581094:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827150.581131:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827150/real 1787827150] req@ffff907d908a21c0 x1874670011551616/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827150.581153:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011551616/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827150.581162:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827150.581166:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827150.581170:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011551616:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827150.581176:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011551744:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827150.581198:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827150.581212:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827150/real 1787827150] req@ffff907d908a2580 x1874670011551744/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827150.581223:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011551744/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827150.581230:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827150.581233:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827150.581237:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011551744:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827150.581242:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011551872:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827150.581261:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827150.581280:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827150/real 1787827150] req@ffff907d908a2940 x1874670011551872/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827150.581291:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011551872/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827150.581305:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827150.581309:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827150.581312:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011551872:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827150.581318:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011552000:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827150.581336:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827150.581346:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827150/real 1787827150] req@ffff907d908a0000 x1874670011552000/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827150.581357:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011552000/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827150.581370:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827150.581389:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827150.581392:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011552000:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827150.708711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827150.708779:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827151.732734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827151.732874:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827152.756815:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827152.756878:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827153.780763:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827153.780829:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827154.804722:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827154.804726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:2.0:1787827155.700795:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827155.700816:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827155.700820:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827155.700829:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2559 00000100:00080000:2.0:1787827155.700840:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827155.700847:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827155.700885:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011552128/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827155.700916:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827155.700921:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827155.700923:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827155.700927:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2559 00000100:00080000:2.0:1787827155.700931:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827155.700935:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827155.700946:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827155.700959:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011552256/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827155.700969:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827155.700973:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827155.700974:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827155.700979:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2559 00000100:00080000:2.0:1787827155.700983:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787827155.700988:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011552128:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827155.700989:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827155.700999:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827155.701015:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011552384/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827155.701026:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827155.701030:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827155.701032:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827155.701037:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2559 00000100:00080000:2.0:1787827155.701043:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827155.701046:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827155.701050:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787827155.701061:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827155.701061:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011552512/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827155.701102:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827155/real 1787827155] req@ffff907d9114b840 x1874670011552128/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827155.701117:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011552128/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827155.701246:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827155.701253:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827155.701257:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011552128:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827155.701264:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011552256:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827155.701290:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827155.701303:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827155/real 1787827155] req@ffff907d9114a580 x1874670011552256/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827155.701318:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011552256/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827155.701327:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827155.701330:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827155.701333:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011552256:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827155.701383:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011552384:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827155.701407:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827155.701424:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827155/real 1787827155] req@ffff907d91149a40 x1874670011552384/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827155.701452:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011552384/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827155.701461:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827155.701464:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827155.701468:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011552384:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827155.701477:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011552512:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827155.701506:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827155.701525:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827155/real 1787827155] req@ffff907d9114ad00 x1874670011552512/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827155.701539:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011552512/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827155.701553:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827155.701556:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827155.701560:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011552512:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827155.828756:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827155.828823:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827156.852735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827156.852799:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827157.876813:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827157.876815:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787827158.900780:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827158.900855:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827159.924747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827159.924833:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827160.822228:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827160.822236:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827160.822241:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827160.822249:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2564 00000100:00080000:3.0:1787827160.822259:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827160.822264:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827160.822298:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011552640/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827160.822340:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827160.822344:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827160.822345:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827160.822350:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2564 00000100:00080000:3.0:1787827160.822354:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827160.822358:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827160.822368:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827160.822380:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011552768/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827160.822390:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827160.822393:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827160.822395:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827160.822398:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2564 00000100:00080000:3.0:1787827160.822402:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827160.822406:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827160.822410:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827160.822419:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011552896/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827160.822433:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827160.822437:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827160.822438:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827160.822441:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2564 00000100:00080000:3.0:1787827160.822445:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827160.822448:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827160.822452:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827160.822462:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011553024/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827160.822603:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011552640:192.168.203.104@tcp:250:kworker.0 00000800:00000100:1.0:1787827160.822682:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827160.822721:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827160/real 1787827160] req@ffff907da562e1c0 x1874670011552640/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827160.822869:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011552640/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827160.822880:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827160.822884:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827160.822888:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011552640:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827160.822895:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011552768:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827160.822920:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827160.822932:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827160/real 1787827160] req@ffff907da562e940 x1874670011552768/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827160.822942:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011552768/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827160.822950:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827160.822952:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827160.822956:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011552768:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827160.822961:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011552896:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827160.822979:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827160.823051:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827160/real 1787827160] req@ffff907da562fc00 x1874670011552896/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827160.823066:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011552896/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827160.823138:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827160.823141:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827160.823144:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011552896:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827160.823150:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011553024:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827160.823173:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827160.823184:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827160/real 1787827160] req@ffff907da562f0c0 x1874670011553024/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827160.823194:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011553024/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827160.823252:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827160.823256:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827160.823259:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011553024:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827160.948772:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827160.949395:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827161.972740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827161.972824:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827162.996756:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827162.996833:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827164.020726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827164.020819:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827165.044745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827165.044822:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827165.940739:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827165.940748:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827165.940752:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827165.940761:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2569 00000100:00080000:2.0:1787827165.940771:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827165.940779:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827165.940813:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011553152/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827165.940840:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827165.940844:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827165.940846:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827165.940850:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2569 00000100:00080000:2.0:1787827165.940854:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827165.940859:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827165.940869:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827165.940883:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011553152:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827165.940884:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011553280/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827165.940899:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827165.940902:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827165.940904:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827165.940908:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2569 00000100:00080000:2.0:1787827165.940912:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827165.940916:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827165.940921:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827165.940933:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011553408/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827165.940940:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787827165.940949:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827165.940953:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827165.940955:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827165.940959:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2569 00000100:00080000:2.0:1787827165.940963:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827165.940968:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827165.940972:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827165/real 1787827165] req@ffff907d9114a940 x1874670011553152/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787827165.940973:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827165.940991:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011553536/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827165.940996:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011553152/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827165.941015:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827165.941018:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827165.941020:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011553152:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827165.941037:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011553280:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827165.941051:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827165.941058:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827165/real 1787827165] req@ffff907d911492c0 x1874670011553280/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827165.941065:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011553280/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827165.941070:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827165.941072:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827165.941074:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011553280:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827165.941077:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011553408:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827165.941086:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827165.941092:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827165/real 1787827165] req@ffff907d91148f00 x1874670011553408/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827165.941101:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011553408/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827165.941116:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827165.941120:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827165.941123:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011553408:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827165.941130:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011553536:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827165.941147:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827165.941157:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827165/real 1787827165] req@ffff907d91148b40 x1874670011553536/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827165.941165:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011553536/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827165.941174:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827165.941176:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827165.941178:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011553536:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827166.068747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827166.068765:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827167.092765:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827167.092792:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827168.116733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827168.116746:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827169.140748:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827169.140838:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827170.164733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827170.164791:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827171.060767:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827171.060777:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827171.060781:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827171.060792:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2574 00000100:00080000:1.0:1787827171.060803:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827171.060810:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827171.060847:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011553664/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827171.060879:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827171.060883:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827171.060885:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827171.060889:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2574 00000100:00080000:1.0:1787827171.060893:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827171.060897:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827171.060906:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827171.060919:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011553792/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827171.060930:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787827171.060932:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011553664:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827171.060935:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827171.060939:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827171.060945:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2574 00000100:00080000:1.0:1787827171.060948:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827171.060952:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827171.060957:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827171.060971:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011553920/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827171.060980:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827171.060983:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827171.060985:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827171.060988:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2574 00000100:00080000:1.0:1787827171.060992:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827171.060996:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:3.0:1787827171.061000:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:1.0:1787827171.061004:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827171.061017:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011554048/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827171.061046:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827171/real 1787827171] req@ffff907d908a1e00 x1874670011553664/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827171.061064:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011553664/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827171.061081:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827171.061084:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827171.061088:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011553664:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827171.061108:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011553792:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827171.061127:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827171.061139:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827171/real 1787827171] req@ffff907d908a2d00 x1874670011553792/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827171.061151:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011553792/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827171.061158:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827171.061161:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827171.061164:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011553792:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827171.061170:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011553920:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827171.061188:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827171.061199:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827171/real 1787827171] req@ffff907d908a0000 x1874670011553920/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827171.061210:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011553920/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827171.061217:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827171.061220:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827171.061223:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011553920:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827171.061229:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011554048:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827171.061258:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827171.061269:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827171/real 1787827171] req@ffff907d908a2940 x1874670011554048/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827171.061279:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011554048/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827171.061287:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827171.061290:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827171.061293:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011554048:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827171.188706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827171.188774:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827172.212762:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827172.212839:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787827173.236715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827173.236777:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827174.260692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827174.260752:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827175.284889:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827175.285048:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:1.0:1787827176.180739:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827176.180749:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827176.180753:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827176.180763:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2579 00000100:00080000:1.0:1787827176.180774:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827176.180781:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827176.180814:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011554176/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827176.180839:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827176.180843:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827176.180844:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827176.180848:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2579 00000100:00080000:1.0:1787827176.180851:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827176.180855:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827176.180864:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827176.180875:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011554176:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827176.180878:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011554304/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827176.180890:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827176.180894:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827176.180896:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827176.180900:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2579 00000100:00080000:1.0:1787827176.180904:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827176.180907:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827176.180913:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827176.180924:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011554432/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827176.180933:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827176.180936:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827176.180940:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1787827176.180940:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787827176.180947:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2579 00000100:00080000:1.0:1787827176.180951:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827176.180955:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827176.180960:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827176.180973:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827176/real 1787827176] req@ffff907d908a2580 x1874670011554176/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827176.180974:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011554560/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827176.181001:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011554176/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827176.181012:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827176.181017:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827176.181021:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011554176:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827176.181043:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011554304:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827176.181065:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827176.181076:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827176/real 1787827176] req@ffff907d908a21c0 x1874670011554304/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827176.181088:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011554304/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827176.181096:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827176.181100:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827176.181103:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011554304:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827176.181109:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011554432:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827176.181123:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827176.181130:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827176/real 1787827176] req@ffff907d908a12c0 x1874670011554432/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827176.181137:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011554432/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827176.181142:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827176.181144:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827176.181145:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011554432:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827176.181148:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011554560:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827176.181156:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827176.181161:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827176/real 1787827176] req@ffff907d908a3840 x1874670011554560/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827176.181168:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011554560/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827176.181173:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827176.181174:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827176.181176:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011554560:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827176.308772:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827176.308842:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827177.332778:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827177.332786:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827178.356738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827178.356788:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827179.380723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827179.381296:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827180.404724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827180.404769:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827181.300765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827181.300773:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827181.300776:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827181.300783:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2585 00000100:00080000:0.0:1787827181.300793:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827181.300798:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827181.300826:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011554688/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827181.300848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827181.300850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827181.300851:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827181.300854:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2585 00000100:00080000:0.0:1787827181.300855:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827181.300857:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827181.300864:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827181.300871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011554816/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827181.300878:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827181.300880:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787827181.300882:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827181.300885:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2585 00000100:00080000:0.0:1787827181.300887:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827181.300889:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827181.300892:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827181.300898:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011554944/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827181.300903:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011554688:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787827181.300906:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827181.300912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827181.300913:0:36:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827181.300916:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2585 00000100:00080000:0.0:1787827181.300918:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827181.300920:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827181.300922:0:36:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827181.300929:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011555072/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827181.300999:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011554816:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827181.301037:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011554944:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827181.301061:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011555072:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827181.428697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827181.428759:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827182.452695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827182.452737:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827183.476694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827183.476720:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787827184.372913:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:0.0:1787827184.372954:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787827184.372971:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787827184.373006:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787827184.373015:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787827184.373023:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:3.0:1787827184.373056:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827181/real 1787827184] req@ffff907d90847480 x1874670011554944/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827184.373086:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011554944/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827184.373100:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827184.373107:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827184.373113:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011554944:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827184.373122:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827181/real 1787827184] req@ffff907d90845a40 x1874670011555072/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827184.373135:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011555072/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827184.373156:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827184.373160:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827184.373164:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011555072:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827184.373173:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827181/real 1787827184] req@ffff907d90844f00 x1874670011554688/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827184.373191:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011554688/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827184.373199:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827184.373202:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827184.373205:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011554688:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827184.373212:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827181/real 1787827184] req@ffff907d90846940 x1874670011554816/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827184.373224:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011554816/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827184.373231:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827184.373235:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827184.373238:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011554816:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827184.500696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827184.500775:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827185.524724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827185.524754:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827186.420734:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827186.420744:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827186.420749:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827186.420760:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2590 00000100:00080000:1.0:1787827186.420771:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827186.420784:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827186.420819:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011555200/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827186.420841:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827186.420844:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827186.420845:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827186.420848:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2590 00000100:00080000:1.0:1787827186.420850:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827186.420852:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827186.420862:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827186.420870:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011555328/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827186.420877:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827186.420880:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827186.420883:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787827186.420886:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011555200:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827186.420890:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2590 00000100:00080000:1.0:1787827186.420895:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827186.420899:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827186.420904:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827186.420918:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011555456/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827186.420933:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827186.420936:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827186.420938:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827186.420941:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2590 00000100:00080000:1.0:1787827186.420946:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827186.420949:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827186.420954:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787827186.420967:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827186.420979:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011555584/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827186.421018:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827186/real 1787827186] req@ffff907d908a0f00 x1874670011555200/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827186.421038:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011555200/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827186.421049:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827186.421053:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827186.421056:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011555200:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827186.421087:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011555328:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827186.421111:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827186.421139:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827186/real 1787827186] req@ffff907d908a1680 x1874670011555328/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827186.421152:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011555328/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827186.421160:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827186.421164:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827186.421167:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011555328:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827186.421174:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011555456:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827186.421198:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827186.421210:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827186/real 1787827186] req@ffff907d908a3840 x1874670011555456/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827186.421221:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011555456/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827186.421229:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827186.421232:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827186.421236:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011555456:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827186.421242:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011555584:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827186.421262:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827186.421273:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827186/real 1787827186] req@ffff907d908a12c0 x1874670011555584/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827186.421286:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011555584/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827186.421295:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827186.421298:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827186.421301:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011555584:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827186.548759:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827186.548812:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827187.572705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827187.572730:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827188.596712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827188.596767:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827189.620684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827189.620761:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827190.644697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827190.644722:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827191.540723:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827191.540731:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827191.540735:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827191.540743:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2595 00000100:00080000:1.0:1787827191.540752:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827191.540759:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827191.540787:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011555712/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827191.540809:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827191.540813:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827191.540815:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827191.540822:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2595 00000100:00080000:1.0:1787827191.540837:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827191.540844:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827191.540855:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827191.540862:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011555712:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827191.540878:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011555840/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827191.540893:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827191.540898:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827191.540900:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827191.540905:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2595 00000100:00080000:1.0:1787827191.540910:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827191.540914:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827191.540922:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787827191.540925:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827191.540944:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011555968/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827191.540957:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827191.540961:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827191.540964:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787827191.540965:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827191/real 1787827191] req@ffff907d908a21c0 x1874670011555712/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827191.540975:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2595 00000100:00080000:1.0:1787827191.540983:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787827191.540987:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011555712/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827191.540990:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827191.541002:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1787827191.541004:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827191.541010:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827191.541014:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011555712:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827191.541022:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011556096/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827191.541039:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011555840:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827191.541061:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827191.541072:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827191/real 1787827191] req@ffff907d908a2580 x1874670011555840/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827191.541102:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011555840/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827191.541110:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827191.541113:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827191.541116:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011555840:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827191.541120:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011555968:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827191.541136:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827191.541146:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827191/real 1787827191] req@ffff907d908a2940 x1874670011555968/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827191.541159:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011555968/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827191.541168:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827191.541171:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827191.541175:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011555968:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827191.541190:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011556096:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827191.541209:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827191.541219:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827191/real 1787827191] req@ffff907d908a0000 x1874670011556096/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827191.541228:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011556096/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827191.541233:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827191.541236:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827191.541238:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011556096:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827191.668676:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827191.668709:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827192.692781:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827192.692831:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827193.716705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827193.716765:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827194.740743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827194.740798:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827195.764727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827195.764791:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827196.660786:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827196.660801:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827196.660806:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827196.660820:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2600 00000100:00080000:3.0:1787827196.660834:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827196.660852:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827196.660903:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011556224/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827196.660930:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827196.660933:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827196.660935:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827196.660938:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2600 00000100:00080000:3.0:1787827196.660942:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827196.660944:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827196.660954:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827196.660963:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011556352/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827196.660972:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827196.660974:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827196.660977:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827196.660980:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2600 00000100:00080000:3.0:1787827196.660983:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827196.660985:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827196.660990:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827196.660997:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011556480/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827196.661006:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827196.661010:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827196.661012:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827196.661016:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2600 00000100:00080000:3.0:1787827196.661020:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787827196.661030:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011556224:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827196.661030:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827196.661041:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827196.661074:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011556608/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827196.661131:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827196.661174:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827196/real 1787827196] req@ffff907da562c3c0 x1874670011556224/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827196.661193:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011556224/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827196.661204:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827196.661209:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827196.661214:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011556224:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827196.661222:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011556352:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827196.661244:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827196.661254:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827196/real 1787827196] req@ffff907da562d2c0 x1874670011556352/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827196.661262:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011556352/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827196.661268:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827196.661270:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827196.661272:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011556352:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827196.661277:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011556480:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827196.661289:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827196.661295:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827196/real 1787827196] req@ffff907da562e580 x1874670011556480/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827196.661303:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011556480/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827196.661308:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827196.661310:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827196.661312:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011556480:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827196.661332:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011556608:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827196.661343:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827196.661349:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827196/real 1787827196] req@ffff907da562c000 x1874670011556608/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827196.661357:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011556608/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827196.661362:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827196.661363:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827196.661365:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011556608:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827196.788704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827196.788759:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787827197.812742:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827197.812743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787827198.836736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827198.836773:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827199.860753:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827199.860819:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827200.884733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827200.884803:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827201.780732:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827201.780744:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827201.780749:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827201.780761:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2605 00000100:00080000:3.0:1787827201.780777:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827201.780786:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827201.780838:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011556736/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827201.780877:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827201.780882:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827201.780885:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827201.780890:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2605 00000100:00080000:3.0:1787827201.780895:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827201.780901:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827201.780912:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827201.780928:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011556864/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827201.780930:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011556736:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827201.780949:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827201.780952:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827201.780954:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827201.780957:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2605 00000100:00080000:3.0:1787827201.780961:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827201.780964:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827201.780968:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827201.780983:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011556992/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827201.780990:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827201.780994:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827201.780995:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827201.780997:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2605 00000100:00080000:3.0:1787827201.781001:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787827201.781005:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787827201.781008:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827201.781013:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827201.781039:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011557120/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827201.781066:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827201/real 1787827201] req@ffff907da562f480 x1874670011556736/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827201.781089:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011556736/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827201.781098:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827201.781102:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827201.781105:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011556736:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827201.781129:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011556864:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827201.781147:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827201.781156:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827201/real 1787827201] req@ffff907da562c780 x1874670011556864/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827201.781164:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011556864/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827201.781171:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827201.781173:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827201.781175:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011556864:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827201.781179:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011556992:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827201.781192:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827201.781198:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827201/real 1787827201] req@ffff907da562ed00 x1874670011556992/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827201.781206:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011556992/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827201.781216:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827201.781219:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827201.781221:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011556992:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827201.781224:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011557120:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827201.781234:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827201.781239:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827201/real 1787827201] req@ffff907da562cf00 x1874670011557120/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827201.781246:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011557120/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827201.781252:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827201.781254:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827201.781256:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011557120:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827201.908717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827201.908797:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827202.932709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827202.932875:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827203.956713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827203.956799:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827204.980705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827204.980804:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827206.004708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827206.004800:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827206.900710:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827206.900721:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827206.900726:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827206.900737:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2610 00000100:00080000:3.0:1787827206.900749:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827206.900755:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827206.900805:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011557248/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827206.900839:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827206.900843:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827206.900845:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827206.900850:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2610 00000100:00080000:3.0:1787827206.900856:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827206.900860:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827206.900873:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827206.900891:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011557376/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827206.900904:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827206.900908:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827206.900910:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787827206.900918:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011557248:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827206.900920:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2610 00000100:00080000:3.0:1787827206.900934:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827206.900943:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827206.900951:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827206.900969:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011557504/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827206.900992:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827206.900997:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827206.900999:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827206.901002:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2610 00000100:00080000:3.0:1787827206.901013:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827206.901017:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:2.0:1787827206.901027:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:3.0:1787827206.901030:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827206.901048:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011557632/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827206.901073:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827206/real 1787827206] req@ffff907da562d680 x1874670011557248/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827206.901092:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011557248/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827206.901107:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827206.901110:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827206.901114:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011557248:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827206.901134:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011557376:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827206.901152:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827206.901161:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827206/real 1787827206] req@ffff907da562cf00 x1874670011557376/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827206.901169:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011557376/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827206.901174:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827206.901177:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827206.901179:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011557376:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827206.901184:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011557504:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827206.901195:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827206.901201:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827206/real 1787827206] req@ffff907da562ed00 x1874670011557504/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827206.901208:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011557504/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827206.901214:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827206.901216:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827206.901218:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011557504:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827206.901222:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011557632:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827206.901232:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827206.901238:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827206/real 1787827206] req@ffff907da562c780 x1874670011557632/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827206.901245:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011557632/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827206.901250:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827206.901252:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827206.901254:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011557632:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827207.028729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827207.028801:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827208.052742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827208.052823:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827209.076690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827209.076733:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827210.100707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827210.100775:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827211.124706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827211.124732:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827212.020689:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827212.020698:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827212.020701:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827212.020710:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2615 00000100:00080000:2.0:1787827212.020720:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827212.020726:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827212.020759:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011557760/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827212.020782:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827212.020785:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827212.020787:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827212.020790:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2615 00000100:00080000:2.0:1787827212.020793:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827212.020797:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827212.020816:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827212.020818:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011557760:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827212.020835:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011557888/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827212.020850:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827212.020854:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827212.020856:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827212.020860:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2615 00000100:00080000:2.0:1787827212.020878:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827212.020882:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827212.020887:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787827212.020891:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827212.020900:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011558016/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827212.020909:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827212.020912:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827212.020914:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827212.020917:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2615 00000100:00080000:2.0:1787827212.020919:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827212.020922:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827212.020925:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827212.020939:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827212/real 1787827212] req@ffff907d91148b40 x1874670011557760/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827212.020940:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011558144/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827212.020981:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011557760/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827212.020991:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827212.020994:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827212.021003:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011557760:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827212.021023:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011557888:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827212.021042:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827212.021052:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827212/real 1787827212] req@ffff907d91148f00 x1874670011557888/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827212.021060:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011557888/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827212.021066:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827212.021068:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827212.021071:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011557888:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827212.021076:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011558016:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827212.021087:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827212.021093:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827212/real 1787827212] req@ffff907d911492c0 x1874670011558016/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827212.021101:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011558016/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827212.021106:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827212.021107:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827212.021109:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011558016:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827212.021113:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011558144:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827212.021122:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827212.021128:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827212/real 1787827212] req@ffff907d9114a940 x1874670011558144/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827212.021135:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011558144/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827212.021140:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827212.021142:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827212.021144:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011558144:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827212.148694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827212.148746:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827213.172686:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827213.172756:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827214.196735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827214.196813:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827215.220686:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827215.220748:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827216.244707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827216.244736:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827217.140732:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827217.140742:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827217.140747:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827217.140757:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2620 00000100:00080000:3.0:1787827217.140768:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827217.140777:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827217.140815:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011558272/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827217.140846:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827217.140850:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827217.140852:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827217.140856:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2620 00000100:00080000:3.0:1787827217.140860:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827217.140864:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827217.140874:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827217.140892:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011558400/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827217.140905:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011558272:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827217.140922:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827217.140933:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827217.140948:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827217.140957:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2620 00000100:00080000:3.0:1787827217.140961:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827217.140966:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827217.140972:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827217.140987:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011558528/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827217.140999:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827217.141004:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827217.141006:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827217.141010:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2620 00000100:00080000:3.0:1787827217.141017:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827217.141023:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827217.141028:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787827217.141035:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827217.141047:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011558656/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827217.141091:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827217/real 1787827217] req@ffff907da562f480 x1874670011558272/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827217.141113:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011558272/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827217.141127:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827217.141132:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827217.141137:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011558272:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827217.141170:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011558400:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827217.141199:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827217.141213:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827217/real 1787827217] req@ffff907da562c000 x1874670011558400/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827217.141226:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011558400/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827217.141235:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827217.141239:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827217.141243:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011558400:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827217.141251:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011558528:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827217.141274:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827217.141285:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827217/real 1787827217] req@ffff907da562e580 x1874670011558528/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827217.141298:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011558528/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827217.141307:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827217.141311:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827217.141314:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011558528:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827217.141320:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011558656:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827217.141339:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827217.141350:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827217/real 1787827217] req@ffff907da562d2c0 x1874670011558656/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827217.141363:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011558656/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827217.141372:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827217.141376:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827217.141380:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011558656:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827217.268705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827217.268788:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827218.292739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827218.292763:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827219.316742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827219.316818:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827220.340699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827220.340804:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827221.364740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827221.364800:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827222.260749:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827222.260759:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827222.260764:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827222.260776:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2626 00000100:00080000:2.0:1787827222.260788:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827222.260795:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827222.260839:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011558784/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827222.260871:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827222.260874:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827222.260887:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827222.260892:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2626 00000100:00080000:2.0:1787827222.260894:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827222.260898:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827222.260908:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827222.260919:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011558912/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827222.260929:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827222.260934:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827222.260937:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787827222.260940:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011558784:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827222.260941:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2626 00000100:00080000:2.0:1787827222.260944:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827222.260950:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827222.260961:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827222.260970:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011559040/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827222.260977:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827222.260979:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827222.260981:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827222.260985:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2626 00000100:00080000:2.0:1787827222.260988:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827222.260991:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827222.260995:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827222.261003:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011559168/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827222.261027:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827222.261066:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827222/real 1787827222] req@ffff907d9114ad00 x1874670011558784/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827222.261081:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011558784/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827222.261094:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827222.261097:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827222.261101:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011558784:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827222.261120:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011558912:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827222.261136:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827222.261148:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827222/real 1787827222] req@ffff907d91149a40 x1874670011558912/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827222.261160:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011558912/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827222.261169:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827222.261173:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827222.261177:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011558912:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827222.261185:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011559040:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827222.261205:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827222.261213:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827222/real 1787827222] req@ffff907d9114a580 x1874670011559040/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827222.261221:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011559040/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827222.261227:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827222.261230:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827222.261232:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011559040:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827222.261237:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011559168:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827222.261247:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827222.261253:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827222/real 1787827222] req@ffff907d9114b840 x1874670011559168/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827222.261260:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011559168/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827222.261265:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827222.261267:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827222.261269:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011559168:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827222.388716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827222.388810:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827223.412733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827223.412788:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827224.436740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827224.436826:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827225.460753:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827225.460782:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827226.484704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827226.484788:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827227.380729:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827227.380741:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827227.380746:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827227.380758:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2631 00000100:00080000:3.0:1787827227.380770:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827227.380777:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827227.380818:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011559296/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827227.380857:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827227.380862:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827227.380864:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827227.380869:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2631 00000100:00080000:3.0:1787827227.380874:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827227.380878:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827227.380890:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827227.380904:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011559424/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827227.380912:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827227.380914:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827227.380916:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827227.380919:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2631 00000100:00080000:3.0:1787827227.380921:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827227.380924:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827227.380928:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827227.380940:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011559552/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827227.380949:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827227.380954:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827227.380956:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827227.380960:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2631 00000100:00080000:3.0:1787827227.380964:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827227.380968:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827227.380975:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011559296:192.168.203.104@tcp:250:kworker.0 00000008:00100000:3.0:1787827227.380977:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827227.381000:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011559680/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827227.381056:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827227.381100:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827227/real 1787827227] req@ffff907da562c3c0 x1874670011559296/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827227.381117:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011559296/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827227.381127:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827227.381132:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827227.381137:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011559296:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827227.381150:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011559424:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827227.381177:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827227.381191:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827227/real 1787827227] req@ffff907da562f0c0 x1874670011559424/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827227.381203:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011559424/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827227.381209:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827227.381212:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827227.381214:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011559424:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827227.381219:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011559552:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827227.381236:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827227.381243:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827227/real 1787827227] req@ffff907da562fc00 x1874670011559552/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827227.381252:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011559552/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827227.381258:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827227.381260:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827227.381262:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011559552:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827227.381291:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011559680:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827227.381314:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827227.381330:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827227/real 1787827227] req@ffff907da562e940 x1874670011559680/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827227.381346:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011559680/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827227.381357:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827227.381361:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827227.381365:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011559680:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827227.508742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827227.508813:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827228.532713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827228.532786:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827229.556709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827229.556807:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827230.580696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827230.580761:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827231.604720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827231.604808:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827232.500735:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827232.500746:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827232.500751:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827232.500761:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2636 00000100:00080000:2.0:1787827232.500773:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827232.500781:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827232.500831:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011559808/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827232.500862:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827232.500867:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827232.500869:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827232.500873:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2636 00000100:00080000:2.0:1787827232.500878:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827232.500885:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827232.500905:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827232.500914:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011559808:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827232.500925:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011559936/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827232.500938:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827232.500942:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827232.500945:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827232.500949:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2636 00000100:00080000:2.0:1787827232.500953:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827232.500957:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827232.500963:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827232.500977:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011560064/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827232.500989:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1787827232.500992:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787827232.500996:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827232.501000:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827232.501006:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2636 00000100:00080000:2.0:1787827232.501018:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827232.501022:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827232.501029:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827232.501034:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827232/real 1787827232] req@ffff907d91149680 x1874670011559808/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827232.501048:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011560192/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827232.501057:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011559808/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827232.501069:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827232.501072:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827232.501080:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011559808:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827232.501102:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011559936:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827232.501126:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827232.501135:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827232/real 1787827232] req@ffff907d9114bc00 x1874670011559936/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827232.501161:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011559936/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827232.501168:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827232.501171:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827232.501173:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011559936:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827232.501178:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011560064:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827232.501192:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827232.501199:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827232/real 1787827232] req@ffff907d911483c0 x1874670011560064/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827232.501215:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011560064/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827232.501220:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827232.501223:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827232.501225:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011560064:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827232.501228:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011560192:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827232.501239:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827232.501245:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827232/real 1787827232] req@ffff907d9114b480 x1874670011560192/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827232.501252:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011560192/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827232.501257:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827232.501259:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827232.501261:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011560192:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827232.628663:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827232.628719:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827233.652761:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827233.652945:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827234.676752:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827234.676830:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827235.700710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827235.700772:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827236.724733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827236.724787:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827237.620709:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827237.620718:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827237.620721:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827237.620730:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2641 00000100:00080000:2.0:1787827237.620740:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827237.620745:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827237.620797:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011560320/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827237.620819:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827237.620822:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827237.620823:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827237.620826:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2641 00000100:00080000:2.0:1787827237.620829:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827237.620832:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827237.620847:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827237.620857:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011560448/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827237.620867:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827237.620870:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827237.620871:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827237.620880:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2641 00000100:00100000:3.0:1787827237.620882:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011560320:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827237.620884:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827237.620895:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827237.620904:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827237.620916:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011560576/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827237.620923:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827237.620926:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827237.620927:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827237.620929:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2641 00000100:00080000:2.0:1787827237.620931:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827237.620933:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827237.620936:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827237.620945:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011560704/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827237.620973:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827237.621017:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827237/real 1787827237] req@ffff907d9114b480 x1874670011560320/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827237.621034:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011560320/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827237.621044:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827237.621047:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827237.621050:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011560320:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827237.621074:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011560448:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827237.621092:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827237.621102:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827237/real 1787827237] req@ffff907d911483c0 x1874670011560448/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827237.621110:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011560448/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827237.621116:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827237.621119:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827237.621121:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011560448:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827237.621125:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011560576:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827237.621139:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827237.621145:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827237/real 1787827237] req@ffff907d9114bc00 x1874670011560576/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827237.621153:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011560576/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827237.621159:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827237.621168:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827237.621170:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011560576:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827237.621175:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011560704:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827237.621187:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827237.621193:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827237/real 1787827237] req@ffff907d91149680 x1874670011560704/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827237.621201:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011560704/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827237.621207:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827237.621208:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827237.621210:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011560704:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827237.748743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827237.748825:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827238.772786:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827238.772839:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827239.796699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827239.796747:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827240.820672:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827240.820757:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827241.844707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827241.844771:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827242.740699:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827242.740707:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827242.740710:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827242.740719:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2646 00000100:00080000:3.0:1787827242.740729:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827242.740747:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827242.740778:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011560832/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827242.740800:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827242.740802:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827242.740804:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827242.740807:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2646 00000100:00080000:3.0:1787827242.740810:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827242.740813:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827242.740821:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827242.740830:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011560960/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827242.740838:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787827242.740840:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011560832:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827242.740841:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827242.740845:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827242.740851:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2646 00000100:00080000:3.0:1787827242.740854:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827242.740856:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827242.740859:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827242.740868:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011561088/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827242.740875:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827242.740878:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827242.740880:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827242.740882:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2646 00000100:00080000:3.0:1787827242.740901:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827242.740905:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827242.740910:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787827242.740911:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827242.740930:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011561216/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827242.740956:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827242/real 1787827242] req@ffff907da562e1c0 x1874670011560832/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827242.740977:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011560832/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827242.740987:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827242.740992:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827242.740995:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011560832:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827242.741013:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011560960:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827242.741032:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827242.741041:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827242/real 1787827242] req@ffff907da562f840 x1874670011560960/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827242.741049:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011560960/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827242.741054:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827242.741057:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827242.741059:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011560960:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827242.741063:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011561088:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827242.741073:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827242.741079:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827242/real 1787827242] req@ffff907da562de00 x1874670011561088/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827242.741086:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011561088/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827242.741091:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827242.741093:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827242.741095:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011561088:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827242.741099:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011561216:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827242.741109:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827242.741114:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827242/real 1787827242] req@ffff907da562cb40 x1874670011561216/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827242.741121:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011561216/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827242.741126:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827242.741128:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827242.741130:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011561216:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827242.868699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827242.868750:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827243.892708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827243.892762:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827244.916677:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827244.916751:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827245.940687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827245.940758:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827246.964755:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827246.964784:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827247.860662:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827247.860677:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827247.860683:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827247.860695:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2651 00000100:00080000:3.0:1787827247.860722:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827247.860732:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827247.860782:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011561344/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827247.860812:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827247.860816:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827247.860818:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827247.860821:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2651 00000100:00080000:3.0:1787827247.860824:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827247.860827:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827247.860837:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827247.860849:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011561472/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827247.860857:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827247.860860:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827247.860862:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827247.860864:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2651 00000100:00080000:3.0:1787827247.860866:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827247.860869:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827247.860874:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827247.860885:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011561600/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827247.860891:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011561344:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827247.860897:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827247.860904:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827247.860905:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827247.860909:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2651 00000100:00080000:3.0:1787827247.860911:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827247.860914:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827247.860919:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827247.860927:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011561728/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827247.860998:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011561472:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827247.861027:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011561600:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827247.861048:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011561728:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827247.988711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827247.988774:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827249.012765:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827249.012861:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827250.036700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827250.036755:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787827250.932853:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787827250.932895:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787827250.932911:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787827250.932947:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787827250.932962:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787827250.932975:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:2.0:1787827250.932997:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827247/real 1787827250] req@ffff907da562de00 x1874670011561600/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827250.933036:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011561600/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827250.933053:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827250.933058:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827250.933062:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011561600:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827250.933069:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827247/real 1787827250] req@ffff907da562f840 x1874670011561728/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827250.933079:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011561728/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827250.933085:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827250.933087:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827250.933089:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011561728:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827250.933094:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827247/real 1787827250] req@ffff907da562da40 x1874670011561344/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827250.933105:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011561344/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827250.933113:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827250.933117:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827250.933121:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011561344:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827250.933129:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827247/real 1787827250] req@ffff907da562cb40 x1874670011561472/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827250.933142:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011561472/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827250.933152:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827250.933156:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827250.933159:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011561472:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827251.060764:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827251.060840:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827252.084697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827252.084745:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827252.980726:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827252.980741:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827252.980748:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827252.980764:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2656 00000100:00080000:3.0:1787827252.980780:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827252.980790:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827252.980838:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011561856/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827252.980884:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827252.980888:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827252.980890:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827252.980896:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2656 00000100:00080000:3.0:1787827252.980900:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827252.980907:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827252.980924:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827252.980930:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011561856:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827252.980956:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011561984/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827252.980977:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827252.980982:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827252.980985:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827252.980991:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2656 00000100:00080000:3.0:1787827252.981004:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827252.981010:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827252.981018:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827252.981036:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011562112/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827252.981047:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827252.981051:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827252.981054:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787827252.981058:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787827252.981068:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2656 00000100:00080000:3.0:1787827252.981077:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827252.981083:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827252.981090:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827252.981105:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011562240/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827252.981128:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827252/real 1787827252] req@ffff907da562e1c0 x1874670011561856/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827252.981154:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011561856/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827252.981167:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827252.981172:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827252.981177:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011561856:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827252.981208:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011561984:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827252.981237:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827252.981252:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827252/real 1787827252] req@ffff907da562e940 x1874670011561984/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827252.981267:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011561984/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827252.981276:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827252.981281:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827252.981285:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011561984:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827252.981309:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011562112:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827252.981332:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827252.981344:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827252/real 1787827252] req@ffff907da562fc00 x1874670011562112/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827252.981357:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011562112/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827252.981367:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827252.981371:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827252.981386:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011562112:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827252.981393:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011562240:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827252.981413:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827252.981424:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827252/real 1787827252] req@ffff907da562f0c0 x1874670011562240/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827252.981437:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011562240/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827252.981446:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827252.981450:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827252.981453:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011562240:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827253.108736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827253.108805:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827254.132695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827254.132713:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827255.156715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827255.156782:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827256.180755:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827256.180776:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827257.204803:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827257.204825:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:3.0:1787827258.100676:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827258.100683:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827258.100688:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827258.100696:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2661 00000100:00080000:3.0:1787827258.100706:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827258.100712:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827258.100742:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011562368/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827258.100763:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827258.100766:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827258.100767:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827258.100771:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2661 00000100:00080000:3.0:1787827258.100775:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827258.100780:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827258.100790:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827258.100805:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011562496/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827258.100811:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011562368:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827258.100826:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827258.100829:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827258.100830:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827258.100835:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2661 00000100:00080000:3.0:1787827258.100838:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827258.100841:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827258.100846:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827258.100857:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011562624/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827258.100863:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827258.100866:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827258.100867:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827258.100869:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2661 00000100:00080000:3.0:1787827258.100871:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827258.100874:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:2.0:1787827258.100880:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:3.0:1787827258.100881:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827258.100891:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011562752/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827258.100921:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827258/real 1787827258] req@ffff907da562c3c0 x1874670011562368/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827258.100936:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011562368/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827258.100944:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827258.100947:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827258.100949:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011562368:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827258.100967:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011562496:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827258.100983:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827258.100991:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827258/real 1787827258] req@ffff907da562d2c0 x1874670011562496/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827258.100999:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011562496/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827258.101004:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827258.101007:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827258.101009:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011562496:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827258.101012:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011562624:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827258.101025:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827258.101031:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827258/real 1787827258] req@ffff907da562e580 x1874670011562624/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827258.101038:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011562624/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827258.101043:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827258.101045:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827258.101046:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011562624:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827258.101050:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011562752:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827258.101059:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827258.101064:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827258/real 1787827258] req@ffff907da562c000 x1874670011562752/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827258.101071:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011562752/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827258.101076:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827258.101078:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827258.101079:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011562752:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827258.228714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827258.228763:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827259.252711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827259.252785:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827260.276737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827260.276806:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827261.300757:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827261.300785:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827262.324758:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827262.324848:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827263.220725:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827263.220734:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827263.220737:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827263.220746:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2666 00000100:00080000:3.0:1787827263.220757:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827263.220763:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827263.220793:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011562880/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827263.220817:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827263.220820:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827263.220821:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827263.220824:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2666 00000100:00080000:3.0:1787827263.220827:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827263.220829:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827263.220836:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827263.220846:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011563008/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827263.220855:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827263.220857:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827263.220859:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827263.220864:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2666 00000100:00100000:2.0:1787827263.220865:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011562880:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827263.220866:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827263.220872:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827263.220879:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827263.220889:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011563136/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827263.220895:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827263.220898:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827263.220899:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827263.220902:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2666 00000100:00080000:3.0:1787827263.220904:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827263.220906:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827263.220910:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827263.220918:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011563264/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827263.220932:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827263.220963:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827263/real 1787827263] req@ffff907da562f480 x1874670011562880/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827263.220975:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011562880/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827263.220984:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827263.220988:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827263.220991:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011562880:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827263.221020:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011563008:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827263.221041:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827263.221051:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827263/real 1787827263] req@ffff907da562c780 x1874670011563008/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827263.221069:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011563008/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827263.221076:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827263.221078:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827263.221080:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011563008:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827263.221085:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011563136:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827263.221113:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827263.221129:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827263/real 1787827263] req@ffff907da562ed00 x1874670011563136/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827263.221150:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011563136/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827263.221160:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827263.221171:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827263.221176:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011563136:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827263.221182:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011563264:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827263.221198:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827263.221209:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827263/real 1787827263] req@ffff907da562cf00 x1874670011563264/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827263.221217:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011563264/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827263.221223:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827263.221225:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827263.221227:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011563264:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827263.348685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827263.348747:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827264.372701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827264.372748:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827265.396693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827265.396763:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827266.420712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827266.420772:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827267.444691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827267.444757:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827268.340681:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827268.340693:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827268.340696:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827268.340705:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2672 00000100:00080000:3.0:1787827268.340714:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827268.340720:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827268.340751:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011563392/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827268.340781:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827268.340785:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827268.340787:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827268.340791:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2672 00000100:00080000:3.0:1787827268.340795:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827268.340800:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827268.340810:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827268.340827:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011563520/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827268.340832:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011563392:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827268.340849:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827268.340851:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827268.340853:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827268.340857:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2672 00000100:00080000:3.0:1787827268.340860:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827268.340863:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827268.340867:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827268.340876:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011563648/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827268.340883:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827268.340885:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827268.340887:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827268.340889:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2672 00000100:00080000:3.0:1787827268.340891:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827268.340893:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827268.340897:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787827268.340898:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827268.340916:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011563776/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827268.340939:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827268/real 1787827268] req@ffff907da562d680 x1874670011563392/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827268.340954:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011563392/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827268.340961:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827268.340964:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827268.340967:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011563392:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827268.340984:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011563520:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827268.340998:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827268.341006:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827268/real 1787827268] req@ffff907da562cf00 x1874670011563520/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827268.341014:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011563520/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827268.341019:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827268.341021:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827268.341024:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011563520:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827268.341027:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011563648:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827268.341037:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827268.341043:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827268/real 1787827268] req@ffff907da562ed00 x1874670011563648/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827268.341050:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011563648/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827268.341059:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827268.341061:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827268.341064:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011563648:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827268.341067:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011563776:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827268.341076:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827268.341081:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827268/real 1787827268] req@ffff907da562c780 x1874670011563776/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827268.341088:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011563776/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827268.341093:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827268.341095:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827268.341097:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011563776:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827268.468744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827268.468781:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827269.492699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827269.492755:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827270.516722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827270.516817:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827271.540694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827271.540779:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827272.564729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827272.564831:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827273.460763:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827273.460774:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827273.460779:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827273.460790:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2677 00000100:00080000:1.0:1787827273.460803:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827273.460809:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827273.460860:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011563904/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827273.460892:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827273.460896:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827273.460897:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827273.460901:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2677 00000100:00080000:1.0:1787827273.460904:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827273.460907:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827273.460918:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827273.460934:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011564032/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827273.460947:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827273.460951:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827273.460953:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827273.460960:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2677 00000100:00100000:2.0:1787827273.460963:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011563904:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827273.460965:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827273.460972:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827273.460978:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827273.460992:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011564160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827273.461000:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827273.461003:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827273.461005:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827273.461008:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2677 00000100:00080000:1.0:1787827273.461010:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827273.461012:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827273.461015:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827273.461024:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011564288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827273.461068:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827273.461136:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827273/real 1787827273] req@ffff907d908a2d00 x1874670011563904/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827273.461157:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011563904/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827273.461177:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827273.461183:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827273.461188:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011563904:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827273.461197:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011564032:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827273.461237:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827273.461251:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827273/real 1787827273] req@ffff907d908a1e00 x1874670011564032/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827273.461264:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011564032/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827273.461290:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827273.461294:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827273.461298:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011564032:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827273.461329:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011564160:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827273.461350:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827273.461361:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827273/real 1787827273] req@ffff907d908a0000 x1874670011564160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827273.461389:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011564160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827273.461398:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827273.461402:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827273.461405:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011564160:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827273.461412:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011564288:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827273.461429:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827273.461439:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827273/real 1787827273] req@ffff907d908a2940 x1874670011564288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827273.461451:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011564288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827273.461460:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827273.461463:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827273.461467:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011564288:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827273.588726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827273.588777:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827274.612714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827274.612803:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827275.636767:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827275.636813:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827276.660750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827276.660822:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827277.684727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827277.684820:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827278.580724:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827278.580735:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827278.580740:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827278.580751:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2682 00000100:00080000:2.0:1787827278.580762:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827278.580769:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827278.580825:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011564416/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827278.580858:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827278.580861:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827278.580864:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827278.580868:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2682 00000100:00080000:2.0:1787827278.580872:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827278.580876:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827278.580888:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827278.580904:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011564416:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827278.580906:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011564544/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827278.580923:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827278.580928:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827278.580930:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827278.580936:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2682 00000100:00080000:2.0:1787827278.580940:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827278.580944:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827278.580949:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827278.580960:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011564672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827278.580967:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827278.580970:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827278.580972:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1787827278.580977:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787827278.580980:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2682 00000100:00080000:2.0:1787827278.580984:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827278.580987:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827278.580991:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827278.581001:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011564800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827278.581034:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827278/real 1787827278] req@ffff907d9114b840 x1874670011564416/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827278.581057:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011564416/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827278.581068:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827278.581071:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827278.581074:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011564416:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827278.581097:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011564544:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827278.581117:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827278.581127:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827278/real 1787827278] req@ffff907d9114a580 x1874670011564544/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827278.581136:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011564544/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827278.581142:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827278.581144:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827278.581146:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011564544:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827278.581152:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011564672:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827278.581165:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827278.581171:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827278/real 1787827278] req@ffff907d91149a40 x1874670011564672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827278.581179:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011564672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827278.581184:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827278.581186:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827278.581188:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011564672:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827278.581192:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011564800:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827278.581210:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827278.581222:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827278/real 1787827278] req@ffff907d9114ad00 x1874670011564800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827278.581233:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011564800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827278.581242:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827278.581246:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827278.581250:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011564800:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827278.708776:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827278.708847:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827279.732706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827279.732793:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827280.756734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827280.756804:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827281.780709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827281.780784:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827282.804710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827282.804747:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827283.700704:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827283.700718:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827283.700725:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827283.700739:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2687 00000100:00080000:3.0:1787827283.700753:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827283.700762:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827283.700824:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011564928/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827283.700852:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827283.700855:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827283.700857:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827283.700860:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2687 00000100:00080000:3.0:1787827283.700863:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827283.700867:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827283.700877:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827283.700887:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011565056/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827283.700896:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827283.700899:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827283.700901:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827283.700906:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2687 00000100:00100000:2.0:1787827283.700908:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011564928:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827283.700909:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827283.700916:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827283.700924:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827283.700936:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011565184/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827283.700947:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827283.700951:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827283.700953:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827283.700958:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2687 00000100:00080000:3.0:1787827283.700979:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827283.700983:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827283.700987:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787827283.700994:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827283.701000:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011565312/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827283.701045:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827283/real 1787827283] req@ffff907da562f480 x1874670011564928/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827283.701062:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011564928/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827283.701072:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827283.701075:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827283.701079:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011564928:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827283.701102:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011565056:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827283.701134:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827283.701144:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827283/real 1787827283] req@ffff907da562c000 x1874670011565056/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827283.701153:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011565056/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827283.701159:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827283.701161:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827283.701164:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011565056:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827283.701169:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011565184:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827283.701182:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827283.701189:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827283/real 1787827283] req@ffff907da562e580 x1874670011565184/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827283.701202:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011565184/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827283.701211:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827283.701215:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827283.701218:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011565184:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827283.701225:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011565312:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827283.701246:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827283.701259:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827283/real 1787827283] req@ffff907da562d2c0 x1874670011565312/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827283.701271:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011565312/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827283.701277:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827283.701279:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827283.701282:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011565312:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827283.828752:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827283.828821:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827284.852676:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827284.852794:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827285.876774:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827285.876868:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827286.900700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827286.900776:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827287.924753:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827287.924836:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827288.820731:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827288.820743:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827288.820748:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827288.820760:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2692 00000100:00080000:3.0:1787827288.820773:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827288.820780:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827288.820818:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011565440/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827288.820846:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827288.820849:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827288.820851:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827288.820854:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2692 00000100:00080000:3.0:1787827288.820856:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827288.820859:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827288.820869:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827288.820899:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011565568/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827288.820910:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787827288.820915:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011565440:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827288.820915:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827288.820918:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827288.820928:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2692 00000100:00080000:3.0:1787827288.820931:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827288.820934:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827288.820938:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827288.820950:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011565696/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827288.820959:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827288.820963:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827288.820964:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827288.820967:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2692 00000100:00080000:3.0:1787827288.820971:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827288.820974:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827288.820979:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827288.820988:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011565824/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827288.821016:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827288.821079:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827288/real 1787827288] req@ffff907da562c3c0 x1874670011565440/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827288.821095:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011565440/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827288.821104:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827288.821108:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827288.821111:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011565440:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827288.821141:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011565568:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827288.821162:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827288.821172:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827288/real 1787827288] req@ffff907da562f0c0 x1874670011565568/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827288.821180:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011565568/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827288.821187:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827288.821189:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827288.821192:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011565568:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827288.821196:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011565696:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827288.821223:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827288.821235:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827288/real 1787827288] req@ffff907da562fc00 x1874670011565696/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827288.821248:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011565696/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827288.821258:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827288.821261:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827288.821265:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011565696:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827288.821271:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011565824:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827288.821287:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827288.821294:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827288/real 1787827288] req@ffff907da562e940 x1874670011565824/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827288.821302:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011565824/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827288.821308:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827288.821310:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827288.821312:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011565824:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827288.948675:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827288.948759:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827289.972684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827289.972765:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827290.996679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827290.996769:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827292.020746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827292.020813:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827293.044701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827293.044754:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827293.940727:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827293.940739:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827293.940744:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827293.940755:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2697 00000100:00080000:2.0:1787827293.940768:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827293.940775:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827293.940825:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011565952/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827293.940852:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827293.940855:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827293.940857:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827293.940860:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2697 00000100:00080000:2.0:1787827293.940862:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827293.940865:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827293.940874:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827293.940886:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011566080/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827293.940893:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827293.940895:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827293.940897:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827293.940900:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2697 00000100:00080000:2.0:1787827293.940903:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827293.940906:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827293.940910:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827293.940921:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011566208/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827293.940925:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011565952:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827293.940930:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827293.940939:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827293.940942:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827293.940946:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2697 00000100:00080000:2.0:1787827293.940948:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827293.940951:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827293.940954:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827293.940961:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011566336/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827293.941029:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827293.941082:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827293/real 1787827293] req@ffff907d9114a940 x1874670011565952/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827293.941103:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011565952/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827293.941115:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827293.941120:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827293.941125:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011565952:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827293.941133:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011566080:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827293.941156:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827293.941167:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827293/real 1787827293] req@ffff907d911492c0 x1874670011566080/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827293.941187:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011566080/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827293.941195:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827293.941199:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827293.941202:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011566080:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827293.941227:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011566208:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827293.941250:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827293.941263:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827293/real 1787827293] req@ffff907d91148f00 x1874670011566208/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827293.941275:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011566208/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827293.941283:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827293.941287:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827293.941291:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011566208:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827293.941297:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011566336:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827293.941316:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827293.941327:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827293/real 1787827293] req@ffff907d91148b40 x1874670011566336/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827293.941339:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011566336/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827293.941354:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827293.941358:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827293.941361:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011566336:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827294.068702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827294.068776:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827295.092716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827295.092756:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827296.116715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827296.116791:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827297.140729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827297.140884:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827298.164741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827298.164803:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827299.060720:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827299.060729:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827299.060734:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827299.060746:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2702 00000100:00080000:2.0:1787827299.060758:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827299.060765:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827299.060810:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011566464/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827299.060838:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827299.060843:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827299.060846:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827299.060850:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2702 00000100:00080000:2.0:1787827299.060858:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827299.060863:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827299.060876:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827299.060891:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011566592/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827299.060901:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827299.060904:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827299.060906:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787827299.060906:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011566464:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827299.060910:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2702 00000100:00080000:2.0:1787827299.060913:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827299.060917:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827299.060922:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827299.060931:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011566720/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827299.060939:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827299.060943:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827299.060944:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827299.060948:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2702 00000100:00080000:2.0:1787827299.060968:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827299.060972:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827299.060976:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787827299.060981:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827299.060988:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011566848/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827299.061029:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827299/real 1787827299] req@ffff907d91148b40 x1874670011566464/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827299.061048:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011566464/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827299.061059:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827299.061063:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827299.061067:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011566464:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827299.061094:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011566592:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827299.061119:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827299.061132:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827299/real 1787827299] req@ffff907d91148f00 x1874670011566592/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827299.061144:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011566592/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827299.061153:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827299.061156:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827299.061159:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011566592:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827299.061165:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011566720:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827299.061185:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827299.061197:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827299/real 1787827299] req@ffff907d911492c0 x1874670011566720/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827299.061209:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011566720/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827299.061217:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827299.061220:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827299.061223:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011566720:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827299.061230:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011566848:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827299.061251:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827299.061264:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827299/real 1787827299] req@ffff907d9114a940 x1874670011566848/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827299.061278:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011566848/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827299.061492:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827299.061496:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827299.061501:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011566848:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827299.188675:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827299.188782:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827300.212726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827300.212776:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827301.236748:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827301.236932:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827302.260727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827302.260798:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827303.284740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827303.284805:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827304.180706:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827304.180713:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827304.180717:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827304.180725:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2707 00000100:00080000:3.0:1787827304.180734:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827304.180740:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827304.180771:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011566976/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827304.180795:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827304.180799:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827304.180801:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827304.180805:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2707 00000100:00080000:3.0:1787827304.180809:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827304.180813:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827304.180823:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827304.180836:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011567104/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827304.180838:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011566976:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827304.180847:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827304.180850:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827304.180852:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827304.180856:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2707 00000100:00080000:3.0:1787827304.180860:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827304.180863:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827304.180867:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827304.180876:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011567232/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827304.180884:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827304.180887:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827304.180889:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787827304.180901:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787827304.180901:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2707 00000100:00080000:3.0:1787827304.180904:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827304.180911:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827304.180915:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827304.180925:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011567360/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827304.180937:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827304/real 1787827304] req@ffff907da562e1c0 x1874670011566976/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827304.180953:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011566976/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827304.180964:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827304.180969:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827304.180973:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011566976:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827304.180994:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011567104:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827304.181013:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827304.181022:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827304/real 1787827304] req@ffff907da562cb40 x1874670011567104/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827304.181031:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011567104/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827304.181037:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827304.181039:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827304.181042:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011567104:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827304.181045:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011567232:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827304.181057:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827304.181063:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827304/real 1787827304] req@ffff907da562da40 x1874670011567232/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827304.181070:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011567232/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827304.181075:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827304.181078:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827304.181080:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011567232:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827304.181083:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011567360:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827304.181092:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827304.181097:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827304/real 1787827304] req@ffff907da562f840 x1874670011567360/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827304.181104:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011567360/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827304.181109:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827304.181111:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827304.181113:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011567360:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827304.308683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827304.308737:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827305.332759:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827305.332832:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827306.356695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827306.356767:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827307.380730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827307.380784:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827308.404677:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827308.404741:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827309.300733:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827309.300743:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827309.300747:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827309.300756:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2713 00000100:00080000:1.0:1787827309.300767:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827309.300774:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827309.300809:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011567488/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827309.300833:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827309.300837:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827309.300838:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827309.300842:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2713 00000100:00080000:1.0:1787827309.300845:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827309.300848:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827309.300857:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827309.300867:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011567616/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827309.300876:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827309.300879:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827309.300881:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827309.300885:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2713 00000100:00080000:1.0:1787827309.300888:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827309.300892:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827309.300897:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827309.300906:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011567488:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827309.300908:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011567744/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827309.300922:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827309.300924:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827309.300926:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827309.300929:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2713 00000100:00080000:1.0:1787827309.300933:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827309.300936:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827309.300941:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827309.300952:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011567872/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827309.300969:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827309.301004:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827309/real 1787827309] req@ffff907d908a2580 x1874670011567488/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827309.301017:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011567488/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827309.301025:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827309.301027:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827309.301030:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011567488:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827309.301035:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011567616:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827309.301049:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827309.301055:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827309/real 1787827309] req@ffff907d908a21c0 x1874670011567616/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827309.301063:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011567616/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827309.301068:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827309.301070:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827309.301072:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011567616:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827309.301089:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011567744:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827309.301100:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827309.301107:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827309/real 1787827309] req@ffff907d908a12c0 x1874670011567744/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827309.301114:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011567744/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827309.301119:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827309.301121:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827309.301122:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011567744:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827309.301126:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011567872:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827309.301135:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827309.301141:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827309/real 1787827309] req@ffff907d908a3840 x1874670011567872/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827309.301147:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011567872/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827309.301152:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827309.301154:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827309.301156:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011567872:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827309.428732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827309.428811:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827310.452702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827310.452764:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827311.476750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827311.476982:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827312.500720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827312.500786:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827313.524709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827313.524760:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827314.420747:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827314.420758:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827314.420762:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827314.420782:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2718 00000100:00080000:2.0:1787827314.420794:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827314.420802:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827314.420845:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011568000/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827314.420888:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827314.420892:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827314.420894:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827314.420899:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2718 00000100:00080000:2.0:1787827314.420903:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827314.420908:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827314.420919:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827314.420936:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011568128/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827314.420937:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011568000:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827314.420951:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827314.420954:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827314.420955:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827314.420959:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2718 00000100:00080000:2.0:1787827314.420962:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827314.420964:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827314.420968:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827314.420978:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011568256/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827314.420985:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827314.420987:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827314.420989:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827314.420993:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2718 00000100:00080000:2.0:1787827314.420997:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827314.421001:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827314.421006:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827314.421018:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011568384/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827314.421046:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011568128:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827314.421069:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011568256:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827314.421083:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011568384:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827314.548711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827314.548767:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827315.572727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827315.572833:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827316.596725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827316.596781:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787827317.492890:0:2337:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:0.0:1787827317.492940:0:2337:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787827317.492961:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787827317.493014:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787827317.493025:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787827317.493037:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:3.0:1787827317.493082:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827314/real 1787827317] req@ffff907d91149a40 x1874670011568128/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827317.493114:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011568128/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827317.493128:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827317.493135:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827317.493140:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011568128:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827317.493150:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827314/real 1787827317] req@ffff907d9114a580 x1874670011568256/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827317.493166:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011568256/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827317.493175:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827317.493179:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827317.493183:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011568256:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827317.493196:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827314/real 1787827317] req@ffff907d9114b840 x1874670011568384/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827317.493209:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011568384/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827317.493232:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827317.493236:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827317.493241:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011568384:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827317.493248:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827314/real 1787827317] req@ffff907d9114ad00 x1874670011568000/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827317.493263:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011568000/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827317.493272:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827317.493276:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827317.493280:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011568000:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827317.620728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827317.620791:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827318.644738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827318.644776:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827319.540753:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827319.540764:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827319.540768:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827319.540779:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2723 00000100:00080000:1.0:1787827319.540791:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827319.540798:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827319.540838:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011568512/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827319.540865:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827319.540868:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827319.540870:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827319.540873:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2723 00000100:00080000:1.0:1787827319.540876:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827319.540881:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827319.540891:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827319.540902:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011568640/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827319.540911:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827319.540914:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827319.540915:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827319.540920:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2723 00000100:00080000:1.0:1787827319.540926:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787827319.540928:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011568512:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827319.540932:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827319.540944:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827319.540965:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011568768/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827319.540976:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827319.540981:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827319.540983:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827319.541010:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2723 00000100:00080000:1.0:1787827319.541017:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827319.541023:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827319.541031:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787827319.541033:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827319.541055:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011568896/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827319.541122:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827319/real 1787827319] req@ffff907d908a1680 x1874670011568512/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827319.541157:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011568512/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827319.541182:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827319.541187:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827319.541192:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011568512:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827319.541234:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011568640:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827319.541265:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827319.541280:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827319/real 1787827319] req@ffff907d908a0f00 x1874670011568640/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827319.541293:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011568640/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827319.541300:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827319.541304:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827319.541307:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011568640:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827319.541315:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011568768:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827319.541336:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827319.541347:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827319/real 1787827319] req@ffff907d908a3840 x1874670011568768/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827319.541359:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011568768/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827319.541368:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827319.541372:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827319.541374:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011568768:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827319.541381:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011568896:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827319.541399:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827319.541410:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827319/real 1787827319] req@ffff907d908a12c0 x1874670011568896/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827319.541422:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011568896/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827319.541429:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827319.541432:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827319.541435:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011568896:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827319.668737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827319.668820:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827320.692763:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827320.692873:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827321.716692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827321.716804:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827322.740709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827322.740807:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827323.764712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827323.764764:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827324.660701:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827324.660721:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827324.660725:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827324.660736:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2728 00000100:00080000:0.0:1787827324.660747:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827324.660754:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827324.660791:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011569024/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827324.660814:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827324.660818:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827324.660820:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827324.660824:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2728 00000100:00080000:0.0:1787827324.660828:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827324.660833:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827324.660843:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827324.660859:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011569152/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827324.660864:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011569024:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787827324.660876:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827324.660881:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787827324.660882:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827324.660887:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2728 00000100:00080000:0.0:1787827324.660890:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827324.660895:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827324.660901:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827324.660913:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011569280/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827324.660924:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827324.660927:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827324.660929:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827324.660933:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2728 00000100:00080000:0.0:1787827324.660937:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827324.660941:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827324.660946:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787827324.660951:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827324.660959:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011569408/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827324.661006:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827324/real 1787827324] req@ffff907d908452c0 x1874670011569024/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827324.661033:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011569024/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827324.661043:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827324.661047:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827324.661052:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011569024:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827324.661075:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011569152:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827324.661100:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827324.661113:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827324/real 1787827324] req@ffff907d90846d00 x1874670011569152/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827324.661122:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011569152/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827324.661128:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827324.661130:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827324.661133:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011569152:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827324.661137:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011569280:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827324.661159:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827324.661184:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827324/real 1787827324] req@ffff907d90845680 x1874670011569280/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827324.661198:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011569280/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827324.661207:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827324.661211:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827324.661215:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011569280:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827324.661221:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011569408:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827324.661241:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827324.661253:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827324/real 1787827324] req@ffff907d90844780 x1874670011569408/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827324.661262:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011569408/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827324.661268:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827324.661270:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827324.661272:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011569408:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827324.788703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827324.788760:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827325.812674:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827325.812720:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827326.836697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827326.836774:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827327.860673:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827327.860758:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827328.884770:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827328.884817:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827329.780740:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827329.780748:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827329.780751:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827329.780760:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2733 00000100:00080000:3.0:1787827329.780770:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827329.780775:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827329.780805:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011569536/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827329.780829:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827329.780832:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827329.780833:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827329.780836:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2733 00000100:00080000:3.0:1787827329.780839:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827329.780841:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827329.780851:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827329.780868:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011569536:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827329.780878:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011569664/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827329.780893:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827329.780897:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827329.780899:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827329.780903:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2733 00000100:00080000:3.0:1787827329.780925:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827329.780932:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:2.0:1787827329.780936:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:3.0:1787827329.780940:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827329.780958:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011569792/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827329.780967:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827329.780970:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827329.780971:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827329.780975:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2733 00000100:00080000:3.0:1787827329.780978:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787827329.780981:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827329/real 1787827329] req@ffff907da562de00 x1874670011569536/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827329.780983:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827329.780987:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827329.780998:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011569920/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827329.781010:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011569536/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827329.781019:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827329.781024:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827329.781027:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011569536:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827329.781050:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011569664:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827329.781069:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827329.781078:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827329/real 1787827329] req@ffff907da562f840 x1874670011569664/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827329.781087:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011569664/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827329.781093:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827329.781095:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827329.781097:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011569664:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827329.781101:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011569792:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827329.781113:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827329.781119:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827329/real 1787827329] req@ffff907da562da40 x1874670011569792/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827329.781140:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011569792/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827329.781151:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827329.781153:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827329.781155:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011569792:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827329.781160:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011569920:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827329.781173:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827329.781181:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827329/real 1787827329] req@ffff907da562cb40 x1874670011569920/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827329.781188:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011569920/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827329.781193:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827329.781195:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827329.781197:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011569920:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827329.908732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827329.908846:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827330.932746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827330.932812:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827331.956753:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827331.956821:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827332.980696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827332.980786:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827334.004711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827334.004762:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827334.900723:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827334.900732:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827334.900737:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827334.900747:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2738 00000100:00080000:3.0:1787827334.900759:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827334.900768:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827334.900802:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011570048/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827334.900834:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827334.900838:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827334.900841:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827334.900846:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2738 00000100:00080000:3.0:1787827334.900851:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827334.900857:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827334.900868:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827334.900882:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011570048:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827334.900889:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011570176/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827334.900901:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827334.900903:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827334.900905:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827334.900908:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2738 00000100:00080000:3.0:1787827334.900911:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827334.900914:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827334.900918:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827334.900930:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011570304/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827334.900943:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827334.900946:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827334.900947:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827334.900951:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2738 00000800:00000100:1.0:1787827334.900955:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787827334.900958:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827334.900963:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827334.900967:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827334.900977:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011570432/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827334.900994:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827334/real 1787827334] req@ffff907da562e1c0 x1874670011570048/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827334.901010:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011570048/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827334.901019:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827334.901022:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827334.901026:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011570048:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827334.901046:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011570176:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827334.901069:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827334.901085:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827334/real 1787827334] req@ffff907da562e940 x1874670011570176/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827334.901118:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011570176/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827334.901127:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827334.901130:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827334.901134:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011570176:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827334.901139:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011570304:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827334.901157:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827334.901165:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827334/real 1787827334] req@ffff907da562fc00 x1874670011570304/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827334.901173:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011570304/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827334.901179:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827334.901181:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827334.901183:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011570304:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827334.901186:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011570432:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827334.901197:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827334.901203:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827334/real 1787827334] req@ffff907da562f0c0 x1874670011570432/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827334.901212:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011570432/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827334.901217:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827334.901219:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827334.901221:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011570432:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827335.028712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827335.028797:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827336.052721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827336.052888:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827337.076731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827337.076803:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827338.100721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827338.100778:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827339.124737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827339.124798:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827340.020716:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827340.020724:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827340.020729:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827340.020738:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2743 00000100:00080000:2.0:1787827340.020746:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827340.020753:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827340.020785:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011570560/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827340.020805:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827340.020808:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827340.020809:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827340.020812:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2743 00000100:00080000:2.0:1787827340.020814:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827340.020817:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827340.020824:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827340.020833:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011570688/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827340.020841:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827340.020844:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827340.020847:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787827340.020848:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011570560:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827340.020851:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2743 00000100:00080000:2.0:1787827340.020855:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827340.020860:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827340.020863:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827340.020870:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011570816/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827340.020882:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827340.020884:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827340.020886:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827340.020888:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2743 00000100:00080000:2.0:1787827340.020890:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827340.020893:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827340.020898:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827340.020909:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011570944/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827340.020925:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827340.020970:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827340/real 1787827340] req@ffff907d91149680 x1874670011570560/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827340.020990:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011570560/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827340.021002:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827340.021007:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827340.021011:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011570560:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827340.021096:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011570688:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827340.021122:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827340.021135:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827340/real 1787827340] req@ffff907d9114bc00 x1874670011570688/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827340.021149:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011570688/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827340.021158:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827340.021162:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827340.021166:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011570688:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827340.021173:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011570816:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827340.021198:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827340.021209:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827340/real 1787827340] req@ffff907d911483c0 x1874670011570816/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827340.021223:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011570816/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827340.021232:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827340.021236:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827340.021239:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011570816:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827340.021246:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011570944:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827340.021266:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827340.021289:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827340/real 1787827340] req@ffff907d9114b480 x1874670011570944/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827340.021303:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011570944/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827340.021312:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827340.021315:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827340.021319:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011570944:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827340.148692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827340.148735:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827341.172669:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827341.172743:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827342.196724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827342.196760:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827343.220685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827343.220757:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827344.244708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827344.244768:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827345.140811:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827345.140821:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827345.140826:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827345.140836:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2748 00000100:00080000:3.0:1787827345.140849:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827345.140857:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827345.140892:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011571072/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827345.140917:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827345.140921:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827345.140923:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827345.140928:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2748 00000100:00080000:3.0:1787827345.140932:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827345.140938:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827345.140949:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827345.140954:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011571072:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827345.140966:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011571200/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827345.140979:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827345.140983:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827345.140985:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827345.140989:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2748 00000100:00080000:3.0:1787827345.140993:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827345.140997:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827345.141003:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827345.141017:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011571328/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827345.141032:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827345.141075:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827345/real 1787827345] req@ffff907da562c3c0 x1874670011571072/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827345.141090:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011571072/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827345.141098:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827345.141101:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827345.141104:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011571072:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827345.141122:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011571200:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827345.141141:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827345.141149:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827345/real 1787827345] req@ffff907da562d2c0 x1874670011571200/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827345.141158:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011571200/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827345.141164:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827345.141166:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827345.141168:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011571200:192.168.203.104@tcp:38:kworker.0 00000100:00080000:3.0:1787827345.141210:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827345.141215:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827345.141217:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827345.141222:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2748 00000100:00080000:3.0:1787827345.141226:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827345.141231:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827345.141234:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011571328:192.168.203.104@tcp:8:kworker.0 00000008:00100000:3.0:1787827345.141239:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827345.141256:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011571456/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827345.141262:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827345.141277:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827345/real 1787827345] req@ffff907da562e580 x1874670011571328/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827345.141291:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011571328/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827345.141298:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827345.141300:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827345.141302:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011571328:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827345.141311:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011571456:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827345.141327:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827345.141337:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827345/real 1787827345] req@ffff907da562c000 x1874670011571456/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827345.141344:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011571456/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827345.141350:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827345.141352:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827345.141354:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011571456:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827345.268684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827345.268732:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827346.292702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827346.292750:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827347.316686:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827347.316750:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827348.340759:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827348.340831:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827349.364689:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827349.364752:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827350.260701:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827350.260708:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827350.260713:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827350.260722:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2754 00000100:00080000:3.0:1787827350.260731:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827350.260738:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827350.260776:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011571584/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827350.260799:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827350.260802:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827350.260803:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827350.260806:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2754 00000100:00080000:3.0:1787827350.260808:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827350.260811:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827350.260819:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827350.260828:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011571712/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827350.260836:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827350.260838:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827350.260840:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827350.260843:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2754 00000100:00080000:3.0:1787827350.260844:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827350.260849:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827350.260852:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011571584:192.168.203.104@tcp:250:kworker.0 00000008:00100000:3.0:1787827350.260853:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827350.260868:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011571840/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827350.260886:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827350.260890:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827350.260892:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827350.260896:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2754 00000100:00080000:3.0:1787827350.260901:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787827350.260915:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787827350.260916:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827350.260924:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827350.260939:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011571968/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827350.260951:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827350/real 1787827350] req@ffff907da562f480 x1874670011571584/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827350.260966:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011571584/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827350.260975:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827350.260979:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827350.260984:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011571584:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827350.261002:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011571712:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827350.261026:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827350.261039:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827350/real 1787827350] req@ffff907da562c780 x1874670011571712/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827350.261050:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011571712/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827350.261057:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827350.261060:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827350.261064:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011571712:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827350.261087:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011571840:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827350.261109:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827350.261120:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827350/real 1787827350] req@ffff907da562ed00 x1874670011571840/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827350.261132:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011571840/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827350.261146:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827350.261149:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827350.261152:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011571840:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827350.261157:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011571968:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827350.261176:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827350.261186:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827350/real 1787827350] req@ffff907da562cf00 x1874670011571968/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827350.261196:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011571968/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827350.261205:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827350.261208:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827350.261211:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011571968:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827350.388694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827350.388769:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827351.412787:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827351.412829:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787827352.436686:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827352.436766:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827353.460705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827353.460798:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827354.484689:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827354.484742:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827355.380719:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827355.380728:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827355.380731:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827355.380739:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2759 00000100:00080000:3.0:1787827355.380748:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827355.380760:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827355.380809:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011572096/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827355.380848:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827355.380852:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827355.380854:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827355.380859:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2759 00000100:00080000:3.0:1787827355.380864:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827355.380868:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827355.380878:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827355.380889:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011572096:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827355.380893:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011572224/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827355.380915:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827355.380917:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827355.380919:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827355.380921:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2759 00000100:00080000:3.0:1787827355.380924:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827355.380926:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827355.380929:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827355.380947:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011572352/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827355.380976:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827355.380979:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827355.380980:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827355.380983:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2759 00000800:00000100:1.0:1787827355.380988:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787827355.380990:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827355.380996:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827355.381000:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827355.381010:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011572480/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827355.381045:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827355/real 1787827355] req@ffff907da562d680 x1874670011572096/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827355.381061:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011572096/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827355.381070:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827355.381073:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827355.381076:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011572096:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827355.381100:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011572224:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827355.381128:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827355.381137:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827355/real 1787827355] req@ffff907da562cf00 x1874670011572224/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827355.381146:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011572224/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827355.381152:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827355.381154:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827355.381156:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011572224:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827355.381160:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011572352:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827355.381172:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827355.381178:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827355/real 1787827355] req@ffff907da562ed00 x1874670011572352/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827355.381185:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011572352/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827355.381191:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827355.381192:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827355.381194:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011572352:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827355.381198:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011572480:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827355.381208:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827355.381214:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827355/real 1787827355] req@ffff907da562c780 x1874670011572480/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827355.381221:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011572480/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827355.381237:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827355.381239:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827355.381241:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011572480:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827355.508725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827355.508793:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827356.532727:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827356.532830:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787827357.556705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827357.556734:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827358.580700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827358.580756:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827359.604728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827359.604818:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827360.500706:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827360.500714:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827360.500729:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827360.500737:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2764 00000100:00080000:3.0:1787827360.500747:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827360.500753:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827360.500801:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011572608/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827360.500822:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827360.500825:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827360.500826:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827360.500828:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2764 00000100:00080000:3.0:1787827360.500831:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827360.500834:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827360.500842:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827360.500850:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011572736/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827360.500857:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827360.500859:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827360.500861:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827360.500863:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2764 00000100:00080000:3.0:1787827360.500864:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827360.500867:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827360.500869:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827360.500874:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011572864/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827360.500885:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827360.500887:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827360.500889:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827360.500891:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2764 00000100:00080000:3.0:1787827360.500893:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827360.500897:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827360.500900:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827360.500908:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011572992/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827360.500982:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011572608:192.168.203.104@tcp:250:kworker.0 00000800:00000100:1.0:1787827360.501071:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827360.501123:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827360/real 1787827360] req@ffff907da562f480 x1874670011572608/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827360.501142:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011572608/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827360.501154:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827360.501158:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827360.501162:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011572608:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827360.501169:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011572736:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827360.501188:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827360.501198:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827360/real 1787827360] req@ffff907da562c000 x1874670011572736/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827360.501208:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011572736/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827360.501231:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827360.501233:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827360.501235:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011572736:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827360.501240:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011572864:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827360.501256:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827360.501263:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827360/real 1787827360] req@ffff907da562e580 x1874670011572864/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827360.501271:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011572864/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827360.501277:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827360.501279:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827360.501281:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011572864:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827360.501285:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011572992:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827360.501296:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827360.501320:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827360/real 1787827360] req@ffff907da562d2c0 x1874670011572992/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827360.501330:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011572992/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827360.501337:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827360.501339:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827360.501341:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011572992:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827360.628745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827360.628815:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827361.652695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827361.652726:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827362.676717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827362.676805:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827363.700765:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827363.700767:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827364.724672:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827364.724703:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827365.620676:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827365.620686:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827365.620691:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827365.620700:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2769 00000100:00080000:3.0:1787827365.620712:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827365.620720:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827365.620759:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011573120/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827365.620788:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827365.620793:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827365.620795:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827365.620800:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2769 00000100:00080000:3.0:1787827365.620804:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827365.620808:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827365.620818:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827365.620832:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011573248/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827365.620845:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787827365.620849:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011573120:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827365.620850:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827365.620855:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827365.620862:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2769 00000100:00080000:3.0:1787827365.620866:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827365.620871:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827365.620877:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827365.620906:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011573376/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827365.620916:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827365.620920:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827365.620922:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827365.620926:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2769 00000100:00080000:3.0:1787827365.620930:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827365.620937:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:1.0:1787827365.620938:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:3.0:1787827365.620946:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827365.620961:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011573504/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827365.620993:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827365/real 1787827365] req@ffff907da562c3c0 x1874670011573120/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827365.621012:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011573120/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827365.621023:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827365.621027:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827365.621031:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011573120:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827365.621052:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011573248:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827365.621088:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827365.621101:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827365/real 1787827365] req@ffff907da562f0c0 x1874670011573248/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827365.621114:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011573248/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827365.621123:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827365.621127:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827365.621130:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011573248:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827365.621137:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011573376:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827365.621160:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827365.621172:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827365/real 1787827365] req@ffff907da562fc00 x1874670011573376/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827365.621185:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011573376/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827365.621193:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827365.621197:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827365.621199:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011573376:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827365.621204:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011573504:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827365.621218:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827365.621226:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827365/real 1787827365] req@ffff907da562e940 x1874670011573504/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827365.621233:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011573504/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827365.621243:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827365.621246:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827365.621248:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011573504:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827365.748706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827365.748760:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827366.772689:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827366.772757:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827367.796741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827367.796802:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827368.820802:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827368.820896:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827369.844794:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827369.844893:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827370.740812:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827370.740825:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827370.740833:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827370.740880:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2774 00000100:00080000:2.0:1787827370.740899:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827370.740911:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827370.740973:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011573632/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827370.741019:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827370.741026:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827370.741029:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827370.741037:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2774 00000100:00080000:2.0:1787827370.741043:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827370.741049:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827370.741065:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827370.741087:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011573760/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827370.741100:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011573632:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827370.741109:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827370.741117:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827370.741120:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827370.741128:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2774 00000100:00080000:2.0:1787827370.741134:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827370.741140:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827370.741147:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827370.741167:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011573888/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827370.741179:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827370.741185:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827370.741188:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827370.741194:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2774 00000100:00080000:2.0:1787827370.741199:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827370.741203:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827370.741209:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787827370.741224:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827370.741231:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011574016/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827370.741444:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827370/real 1787827370] req@ffff907d9114b480 x1874670011573632/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827370.741734:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011573632/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827370.741753:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827370.741760:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827370.741766:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011573632:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827370.741931:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011573760:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827370.741973:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827370.741994:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827370/real 1787827370] req@ffff907d911483c0 x1874670011573760/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827370.742013:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011573760/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827370.742025:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827370.742029:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827370.742034:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011573760:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827370.742042:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011573888:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827370.742065:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827370.742078:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827370/real 1787827370] req@ffff907d9114bc00 x1874670011573888/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827370.742093:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011573888/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827370.742113:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827370.742117:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827370.742121:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011573888:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827370.742128:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011574016:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827370.742147:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827370.742158:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827370/real 1787827370] req@ffff907d91149680 x1874670011574016/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827370.742172:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011574016/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827370.742182:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827370.742186:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827370.742189:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011574016:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827370.868766:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827370.868884:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827371.892754:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827371.892780:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827372.916743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827372.916868:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827373.940693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827373.940846:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827374.964700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827374.964749:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827375.860709:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827375.860720:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827375.860724:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827375.860738:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2779 00000100:00080000:2.0:1787827375.860753:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827375.860764:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827375.860839:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011574144/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827375.860870:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827375.860873:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827375.860875:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827375.860878:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2779 00000100:00080000:2.0:1787827375.860882:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827375.860885:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827375.860895:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827375.860907:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011574272/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827375.860917:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787827375.860919:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011574144:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827375.860920:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827375.860924:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827375.860932:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2779 00000100:00080000:2.0:1787827375.860935:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827375.860937:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827375.860941:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827375.860950:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011574400/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827375.860957:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827375.860960:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827375.860962:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827375.860966:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2779 00000100:00080000:2.0:1787827375.860990:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827375.860993:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827375.860998:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787827375.861002:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827375.861012:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011574528/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827375.861049:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827375/real 1787827375] req@ffff907d9114ad00 x1874670011574144/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827375.861064:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011574144/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827375.861081:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827375.861085:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827375.861090:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011574144:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827375.861118:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011574272:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827375.861140:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827375.861150:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827375/real 1787827375] req@ffff907d9114b840 x1874670011574272/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827375.861159:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011574272/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827375.861165:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827375.861167:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827375.861169:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011574272:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827375.861173:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011574400:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827375.861186:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827375.861192:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827375/real 1787827375] req@ffff907d9114a580 x1874670011574400/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827375.861199:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011574400/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827375.861205:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827375.861207:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827375.861208:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011574400:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827375.861213:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011574528:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827375.861222:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827375.861228:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827375/real 1787827375] req@ffff907d91149a40 x1874670011574528/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827375.861235:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011574528/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827375.861241:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827375.861242:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827375.861244:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011574528:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827375.988707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827375.988847:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827377.012723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827377.012840:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827378.036779:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827378.036791:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787827379.060695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827379.060712:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827380.084713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827380.084813:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827380.980682:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827380.980690:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827380.980694:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827380.980702:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2784 00000100:00080000:3.0:1787827380.980712:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827380.980718:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827380.980751:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011574656/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827380.980786:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827380.980790:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827380.980792:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827380.980795:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2784 00000100:00080000:3.0:1787827380.980798:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827380.980802:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827380.980812:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827380.980824:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011574656:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827380.980825:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011574784/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827380.980839:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827380.980841:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827380.980842:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827380.980845:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2784 00000100:00080000:3.0:1787827380.980848:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827380.980850:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827380.980854:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827380.980863:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011574912/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827380.980870:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827380.980872:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827380.980874:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827380.980876:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2784 00000100:00080000:3.0:1787827380.980879:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827380.980881:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827380.980884:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827380.980893:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011575040/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827380.980925:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011574784:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827380.980949:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011574912:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827380.980965:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011575040:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827381.108698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827381.108746:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827382.132688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827382.132731:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827383.156687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827383.156816:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787827384.052998:0:2338:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:2.0:1787827384.053052:0:2338:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787827384.053072:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787827384.053133:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787827384.053146:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787827384.053157:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:1.0:1787827384.053270:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827380/real 1787827384] req@ffff907da562cb40 x1874670011574784/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827384.053308:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011574784/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827384.053333:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827384.053340:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827384.053346:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011574784:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827384.053368:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827380/real 1787827384] req@ffff907da562da40 x1874670011574912/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827384.053383:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011574912/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827384.053394:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827384.053399:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827384.053404:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011574912:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827384.053413:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827380/real 1787827384] req@ffff907da562f840 x1874670011575040/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827384.053428:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011575040/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827384.053439:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827384.053443:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827384.053447:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011575040:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827384.053456:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827380/real 1787827384] req@ffff907da562e1c0 x1874670011574656/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827384.053486:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011574656/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827384.053498:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827384.053502:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827384.053506:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011574656:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827384.180709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827384.180800:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827385.204723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827385.204862:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827386.100776:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827386.100804:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827386.100808:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827386.100818:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2789 00000100:00080000:3.0:1787827386.100827:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827386.100836:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827386.100877:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011575168/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827386.100916:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827386.100921:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827386.100923:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827386.100928:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2789 00000100:00080000:3.0:1787827386.100933:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827386.100939:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827386.100950:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827386.100968:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011575168:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827386.100973:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011575296/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827386.101022:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827386.101028:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827386.101032:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827386.101039:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2789 00000100:00080000:3.0:1787827386.101045:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827386.101051:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827386.101060:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827386.101095:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011575424/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827386.101105:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787827386.101117:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827386.101121:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827386.101124:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827386.101131:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2789 00000100:00080000:3.0:1787827386.101135:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827386.101141:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827386.101146:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827386.101162:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011575552/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827386.101171:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827386/real 1787827386] req@ffff907da562de00 x1874670011575168/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827386.101206:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011575168/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827386.101221:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827386.101225:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827386.101229:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011575168:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827386.101265:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011575296:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827386.101299:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827386.101317:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827386/real 1787827386] req@ffff907da562e1c0 x1874670011575296/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827386.101334:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011575296/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827386.101345:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827386.101349:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827386.101353:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011575296:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827386.101361:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011575424:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827386.101387:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827386.101403:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827386/real 1787827386] req@ffff907da562f840 x1874670011575424/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827386.101428:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011575424/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827386.101439:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827386.101443:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827386.101448:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011575424:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827386.101456:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011575552:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827386.101481:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827386.101498:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827386/real 1787827386] req@ffff907da562da40 x1874670011575552/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827386.101514:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011575552/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827386.101525:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827386.101529:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827386.101534:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011575552:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827386.228680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827386.228723:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827387.252702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827387.252766:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827388.276710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827388.276794:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827389.300803:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827389.300864:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827390.324739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827390.324805:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827391.220772:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827391.220785:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827391.220791:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827391.220804:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2794 00000100:00080000:2.0:1787827391.220818:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827391.220828:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827391.220876:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011575680/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827391.220917:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827391.220923:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827391.220926:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827391.220931:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2794 00000100:00080000:2.0:1787827391.220936:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827391.220941:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827391.220952:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827391.220969:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011575808/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827391.220984:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827391.220989:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827391.220991:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827391.220997:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2794 00000100:00080000:2.0:1787827391.221000:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827391.221005:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827391.221012:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827391.221016:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011575680:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827391.221030:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011575936/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827391.221047:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827391.221052:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827391.221054:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827391.221060:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2794 00000100:00080000:2.0:1787827391.221065:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827391.221070:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827391.221077:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827391.221092:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011576064/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827391.221113:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827391.221197:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827391/real 1787827391] req@ffff907d9114a940 x1874670011575680/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827391.221235:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011575680/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827391.221249:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827391.221255:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827391.221260:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011575680:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827391.221271:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011575808:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827391.221305:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827391.221322:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827391/real 1787827391] req@ffff907d911492c0 x1874670011575808/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827391.221337:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011575808/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827391.221348:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827391.221352:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827391.221356:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011575808:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827391.221385:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011575936:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827391.221406:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827391.221419:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827391/real 1787827391] req@ffff907d91148f00 x1874670011575936/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827391.221442:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011575936/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827391.221452:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827391.221456:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827391.221459:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011575936:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827391.221466:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011576064:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827391.221484:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827391.221495:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827391/real 1787827391] req@ffff907d91148b40 x1874670011576064/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827391.221509:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011576064/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827391.221519:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827391.221522:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827391.221525:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011576064:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827391.348763:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827391.348938:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827392.372773:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827392.372842:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827393.396716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827393.396767:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827394.420837:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827394.420940:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827395.444697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827395.444780:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827396.340688:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827396.340699:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827396.340704:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827396.340714:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2800 00000100:00080000:2.0:1787827396.340724:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827396.340732:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827396.340766:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011576192/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827396.340790:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827396.340793:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827396.340798:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827396.340801:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2800 00000100:00080000:2.0:1787827396.340804:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827396.340808:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827396.340819:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827396.340835:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011576320/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827396.340838:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011576192:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827396.340855:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827396.340859:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827396.340861:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827396.340865:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2800 00000100:00080000:2.0:1787827396.340868:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827396.340872:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827396.340876:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827396.340885:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011576448/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827396.340893:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827396.340895:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827396.340896:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827396.340898:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2800 00000100:00080000:2.0:1787827396.340901:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787827396.340905:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787827396.340906:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827396.340912:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827396.340919:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011576576/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827396.340944:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827396/real 1787827396] req@ffff907d91148b40 x1874670011576192/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827396.340959:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011576192/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827396.340971:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827396.340974:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827396.340977:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011576192:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827396.340995:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011576320:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827396.341009:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827396.341017:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827396/real 1787827396] req@ffff907d91148f00 x1874670011576320/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827396.341025:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011576320/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827396.341030:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827396.341033:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827396.341035:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011576320:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827396.341038:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011576448:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827396.341048:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827396.341054:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827396/real 1787827396] req@ffff907d911492c0 x1874670011576448/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827396.341061:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011576448/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827396.341067:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827396.341069:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827396.341070:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011576448:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827396.341074:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011576576:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827396.341084:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827396.341089:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827396/real 1787827396] req@ffff907d9114a940 x1874670011576576/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827396.341096:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011576576/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827396.341101:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827396.341103:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827396.341105:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011576576:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827396.468750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827396.468817:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827397.492695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827397.492759:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827398.516786:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827398.516893:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827399.540738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827399.540830:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827400.564727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827400.564812:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827401.460741:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827401.460749:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827401.460753:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827401.460762:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2805 00000100:00080000:3.0:1787827401.460773:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827401.460781:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827401.460814:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011576704/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827401.460835:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827401.460838:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827401.460843:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827401.460847:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2805 00000100:00080000:3.0:1787827401.460849:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827401.460851:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827401.460860:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827401.460869:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011576832/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827401.460883:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787827401.460887:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011576704:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827401.460888:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827401.460892:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827401.460901:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2805 00000100:00080000:3.0:1787827401.460905:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827401.460910:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827401.460916:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827401.460931:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011576960/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827401.460942:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827401.460945:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827401.460946:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827401.460949:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2805 00000800:00000100:1.0:1787827401.460953:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787827401.460955:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827401.460960:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827401.460965:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827401.460976:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011577088/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827401.460996:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827401/real 1787827401] req@ffff907da562cb40 x1874670011576704/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827401.461012:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011576704/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827401.461020:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827401.461023:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827401.461026:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011576704:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827401.461044:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011576832:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827401.461060:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827401.461068:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827401/real 1787827401] req@ffff907da562e940 x1874670011576832/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827401.461077:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011576832/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827401.461083:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827401.461085:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827401.461087:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011576832:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827401.461091:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011576960:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827401.461101:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827401.461107:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827401/real 1787827401] req@ffff907da562fc00 x1874670011576960/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827401.461115:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011576960/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827401.461120:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827401.461122:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827401.461124:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011576960:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827401.461127:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011577088:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827401.461137:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827401.461142:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827401/real 1787827401] req@ffff907da562f0c0 x1874670011577088/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827401.461149:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011577088/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827401.461164:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827401.461167:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827401.461170:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011577088:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827401.588754:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827401.588805:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827402.612769:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827402.612796:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827403.636763:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827403.636862:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827404.660724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827404.660791:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827405.684675:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827405.684746:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827406.580652:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827406.580660:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827406.580663:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827406.580672:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2810 00000100:00080000:3.0:1787827406.580687:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827406.580693:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827406.580725:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011577216/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827406.580745:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827406.580747:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827406.580749:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827406.580751:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2810 00000100:00080000:3.0:1787827406.580754:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827406.580756:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827406.580764:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827406.580773:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011577344/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827406.580780:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827406.580783:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827406.580784:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827406.580787:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2810 00000100:00080000:3.0:1787827406.580789:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827406.580791:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827406.580794:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827406.580796:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011577216:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827406.580803:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011577472/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827406.580811:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827406.580813:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827406.580814:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827406.580816:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2810 00000100:00080000:3.0:1787827406.580818:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827406.580820:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827406.580823:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827406.580830:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011577600/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827406.580859:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827406.580895:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827406/real 1787827406] req@ffff907da562c3c0 x1874670011577216/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827406.580909:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011577216/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827406.580918:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827406.580922:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827406.580925:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011577216:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827406.580930:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011577344:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827406.580952:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827406.580961:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827406/real 1787827406] req@ffff907da562d2c0 x1874670011577344/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827406.580969:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011577344/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827406.580975:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827406.580978:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827406.580980:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011577344:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827406.581012:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011577472:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827406.581030:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827406.581038:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827406/real 1787827406] req@ffff907da562e580 x1874670011577472/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827406.581046:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011577472/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827406.581053:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827406.581056:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827406.581059:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011577472:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827406.581064:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011577600:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827406.581076:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827406.581084:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827406/real 1787827406] req@ffff907da562c000 x1874670011577600/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827406.581092:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011577600/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827406.581098:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827406.581100:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827406.581103:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011577600:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827406.708714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827406.708782:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827407.732747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827407.732842:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827408.756699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827408.756860:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827409.780848:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827409.780888:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787827410.804725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827410.804995:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827411.700820:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827411.700836:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827411.700843:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827411.700876:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2815 00000100:00080000:3.0:1787827411.700893:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827411.700903:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827411.700964:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011577728/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827411.700996:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827411.701001:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827411.701003:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827411.701007:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2815 00000100:00080000:3.0:1787827411.701011:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827411.701014:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827411.701026:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827411.701042:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011577856/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827411.701053:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827411.701057:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827411.701058:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827411.701062:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2815 00000100:00080000:3.0:1787827411.701084:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827411.701091:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827411.701100:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011577728:192.168.203.104@tcp:250:kworker.0 00000008:00100000:3.0:1787827411.701100:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827411.701124:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011577984/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827411.701135:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827411.701140:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827411.701143:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827411.701147:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2815 00000100:00080000:3.0:1787827411.701151:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827411.701155:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827411.701160:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827411.701179:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011578112/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827411.701224:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827411.701300:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827411/real 1787827411] req@ffff907da562f480 x1874670011577728/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827411.701326:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011577728/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827411.701343:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827411.701349:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827411.701355:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011577728:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827411.701368:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011577856:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827411.701414:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827411.701428:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827411/real 1787827411] req@ffff907da562c780 x1874670011577856/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827411.701440:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011577856/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827411.701448:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827411.701452:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827411.701455:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011577856:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827411.701482:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011577984:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827411.701501:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827411.701511:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827411/real 1787827411] req@ffff907da562ed00 x1874670011577984/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827411.701523:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011577984/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827411.701538:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827411.701541:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827411.701544:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011577984:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827411.701550:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011578112:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827411.701593:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827411.701619:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827411/real 1787827411] req@ffff907da562cf00 x1874670011578112/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827411.701648:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011578112/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827411.701654:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827411.701656:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827411.701658:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011578112:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827411.828765:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827411.828955:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787827412.852870:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827412.853301:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827413.876963:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827413.876973:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827414.900746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827414.901218:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827415.924930:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827415.924956:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:3.0:1787827416.820835:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827416.820851:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827416.820858:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827416.820887:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2820 00000100:00080000:3.0:1787827416.820921:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827416.820931:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827416.820988:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011578240/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827416.821020:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827416.821024:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827416.821027:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827416.821032:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2820 00000100:00080000:3.0:1787827416.821048:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827416.821052:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827416.821064:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827416.821082:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011578368/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827416.821092:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011578240:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827416.821099:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827416.821103:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827416.821105:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827416.821109:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2820 00000100:00080000:3.0:1787827416.821113:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827416.821117:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827416.821122:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827416.821133:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011578496/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827416.821141:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827416.821144:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827416.821146:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827416.821149:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2820 00000100:00080000:3.0:1787827416.821152:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827416.821156:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827416.821160:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827416.821169:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011578624/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827416.821177:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827416.821238:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827416/real 1787827416] req@ffff907da562d680 x1874670011578240/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827416.821256:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011578240/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827416.821264:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827416.821268:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827416.821271:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011578240:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827416.821298:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011578368:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827416.821325:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827416.821336:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827416/real 1787827416] req@ffff907da562cf00 x1874670011578368/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827416.821345:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011578368/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827416.821350:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827416.821353:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827416.821355:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011578368:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827416.821359:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011578496:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827416.821371:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827416.821377:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827416/real 1787827416] req@ffff907da562ed00 x1874670011578496/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827416.821385:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011578496/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827416.821390:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827416.821392:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827416.821393:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011578496:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827416.821397:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011578624:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827416.821407:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827416.821414:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827416/real 1787827416] req@ffff907da562c780 x1874670011578624/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827416.821422:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011578624/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827416.821427:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827416.821429:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827416.821430:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011578624:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827416.948766:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827416.948892:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827417.972724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827417.972958:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827418.996736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827418.996848:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827420.020776:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827420.020860:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827421.044725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827421.044866:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827421.940953:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827421.940986:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827421.940994:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827421.941028:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2825 00000100:00080000:0.0:1787827421.941047:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827421.941061:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827421.941148:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011578752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827421.941188:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827421.941193:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827421.941196:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827421.941202:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2825 00000100:00080000:0.0:1787827421.941208:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827421.941214:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827421.941226:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827421.941244:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011578880/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827421.941258:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827421.941285:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787827421.941287:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827421.941290:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2825 00000100:00080000:0.0:1787827421.941295:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827421.941299:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827421.941304:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827421.941313:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011579008/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827421.941322:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011578752:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787827421.941325:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827421.941328:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827421.941330:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827421.941333:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2825 00000100:00080000:0.0:1787827421.941335:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827421.941338:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827421.941341:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827421.941350:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011579136/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827421.941448:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827421.941513:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827421/real 1787827421] req@ffff907d90844000 x1874670011578752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827421.941531:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011578752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827421.941542:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827421.941547:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827421.941553:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011578752:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827421.941563:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011578880:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827421.941592:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827421.941636:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827421/real 1787827421] req@ffff907d908443c0 x1874670011578880/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827421.941647:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011578880/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827421.941659:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827421.941662:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827421.941665:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011578880:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827421.941715:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011579008:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827421.941736:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827421.941746:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827421/real 1787827421] req@ffff907d90844b40 x1874670011579008/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827421.941755:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011579008/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827421.941762:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827421.941764:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827421.941767:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011579008:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827421.941770:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011579136:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827421.941782:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827421.941789:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827421/real 1787827421] req@ffff907d90845e00 x1874670011579136/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827421.941797:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011579136/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827421.941803:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827421.941805:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827421.941808:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011579136:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827422.068724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827422.068923:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827423.092672:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827423.092734:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787827424.116741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827424.116843:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827425.140779:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827425.140807:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827426.164744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827426.164839:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827427.060825:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827427.060846:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827427.060851:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827427.060883:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2830 00000100:00080000:3.0:1787827427.060894:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827427.060901:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827427.060937:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011579264/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827427.060963:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827427.060966:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827427.060968:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827427.060982:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2830 00000100:00080000:3.0:1787827427.060986:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827427.060990:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827427.061027:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827427.061042:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011579264:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827427.061045:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011579392/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827427.061064:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827427.061073:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827427.061075:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827427.061080:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2830 00000100:00080000:3.0:1787827427.061085:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827427.061107:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827427.061111:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827427.061120:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011579520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827427.061127:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827427.061130:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827427.061131:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787827427.061136:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787827427.061139:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2830 00000100:00080000:3.0:1787827427.061143:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827427.061146:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827427.061149:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827427.061159:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011579648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827427.061209:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827427/real 1787827427] req@ffff907da562f480 x1874670011579264/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827427.061259:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011579264/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827427.061269:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827427.061273:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827427.061276:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011579264:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827427.061296:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011579392:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827427.061351:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827427.061396:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827427/real 1787827427] req@ffff907da562c000 x1874670011579392/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827427.061431:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011579392/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827427.061437:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827427.061439:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827427.061442:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011579392:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827427.061446:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011579520:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827427.061478:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827427.061487:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827427/real 1787827427] req@ffff907da562e580 x1874670011579520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827427.061495:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011579520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827427.061501:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827427.061503:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827427.061505:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011579520:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827427.061508:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011579648:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827427.061519:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827427.061525:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827427/real 1787827427] req@ffff907da562d2c0 x1874670011579648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827427.061532:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011579648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827427.061537:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827427.061538:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827427.061540:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011579648:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827427.188694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827427.188792:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827428.212685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827428.212738:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827429.236697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827429.236754:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827430.260695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827430.260737:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827431.284750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827431.284897:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827432.180734:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827432.180743:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827432.180747:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827432.180756:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2835 00000100:00080000:1.0:1787827432.180767:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827432.180774:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827432.180810:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011579776/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827432.180856:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827432.180859:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827432.180861:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827432.180871:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2835 00000100:00080000:1.0:1787827432.180875:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827432.180878:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827432.180888:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827432.180917:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011579904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827432.180927:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827432.180933:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1787827432.180933:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011579776:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827432.180937:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827432.180946:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2835 00000100:00080000:1.0:1787827432.180950:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827432.180952:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827432.180957:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827432.180986:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011580032/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827432.180995:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827432.180998:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827432.180999:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827432.181003:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2835 00000100:00080000:1.0:1787827432.181006:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827432.181010:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827432.181015:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827432.181024:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011580160/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827432.181053:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827432.181095:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827432/real 1787827432] req@ffff907d908a21c0 x1874670011579776/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827432.181117:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011579776/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827432.181126:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827432.181129:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827432.181132:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011579776:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827432.181161:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011579904:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827432.181182:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827432.181193:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827432/real 1787827432] req@ffff907d908a2580 x1874670011579904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827432.181202:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011579904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827432.181211:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827432.181216:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827432.181220:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011579904:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827432.181227:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011580032:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827432.181270:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827432.181286:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827432/real 1787827432] req@ffff907d908a2940 x1874670011580032/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827432.181303:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011580032/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827432.181316:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827432.181319:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827432.181330:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011580032:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827432.181336:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011580160:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827432.181354:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827432.181363:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827432/real 1787827432] req@ffff907d908a0000 x1874670011580160/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827432.181372:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011580160/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827432.181379:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827432.181381:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827432.181384:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011580160:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827432.308694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827432.308741:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827433.332725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827433.332813:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827434.356678:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827434.356731:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827435.380764:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827435.380857:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827436.404685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827436.404775:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827437.300698:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827437.300709:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827437.300713:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827437.300724:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2841 00000100:00080000:1.0:1787827437.300736:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827437.300744:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827437.300780:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011580288/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827437.300805:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827437.300808:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827437.300809:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827437.300812:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2841 00000100:00080000:1.0:1787827437.300815:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827437.300818:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827437.300826:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827437.300838:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011580416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827437.300864:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827437.300868:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827437.300870:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827437.300874:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2841 00000100:00080000:1.0:1787827437.300877:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827437.300882:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827437.300888:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827437.300909:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011580288:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827437.300927:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011580544/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827437.300936:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827437.300939:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827437.300940:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827437.300943:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2841 00000100:00080000:1.0:1787827437.300946:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827437.300949:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827437.300952:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827437.300966:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011580672/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827437.300974:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827437.301044:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827437/real 1787827437] req@ffff907d908a1e00 x1874670011580288/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827437.301066:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011580288/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827437.301073:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827437.301077:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827437.301080:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011580288:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827437.301085:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011580416:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827437.301103:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827437.301112:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827437/real 1787827437] req@ffff907d908a2d00 x1874670011580416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827437.301119:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011580416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827437.301125:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827437.301127:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827437.301129:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011580416:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827437.301146:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011580544:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827437.301158:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827437.301164:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827437/real 1787827437] req@ffff907d908a0000 x1874670011580544/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827437.301171:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011580544/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827437.301180:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827437.301183:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827437.301185:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011580544:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827437.301188:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011580672:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827437.301196:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827437.301202:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827437/real 1787827437] req@ffff907d908a2940 x1874670011580672/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827437.301209:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011580672/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827437.301214:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827437.301215:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827437.301217:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011580672:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827437.428706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827437.428824:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827438.452743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827438.452811:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827439.476758:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827439.476980:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827440.500685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827440.500731:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827441.524722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827441.524799:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827442.420709:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827442.420721:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827442.420727:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827442.420745:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2846 00000100:00080000:1.0:1787827442.420758:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827442.420767:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827442.420806:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011580800/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827442.420842:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827442.420846:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827442.420848:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827442.420852:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2846 00000100:00080000:1.0:1787827442.420856:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827442.420860:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827442.420871:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827442.420886:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011580800:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827442.420890:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011580928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827442.420907:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827442.420910:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827442.420912:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827442.420917:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2846 00000100:00080000:1.0:1787827442.420920:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827442.420923:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827442.420928:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827442.420941:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011581056/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827442.420949:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827442.420953:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827442.420954:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827442.420957:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2846 00000100:00080000:1.0:1787827442.420961:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787827442.420964:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787827442.420968:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827442.420974:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827442.420988:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011581184/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827442.421003:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827442/real 1787827442] req@ffff907d908a2580 x1874670011580800/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827442.421018:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011580800/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827442.421028:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827442.421032:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827442.421036:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011580800:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827442.421055:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011580928:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827442.421072:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827442.421081:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827442/real 1787827442] req@ffff907d908a21c0 x1874670011580928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827442.421090:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011580928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827442.421096:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827442.421099:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827442.421103:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011580928:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827442.421108:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011581056:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827442.421121:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827442.421129:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827442/real 1787827442] req@ffff907d908a12c0 x1874670011581056/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827442.421137:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011581056/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827442.421142:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827442.421144:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827442.421146:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011581056:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827442.421150:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011581184:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827442.421160:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827442.421166:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827442/real 1787827442] req@ffff907d908a3840 x1874670011581184/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827442.421176:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011581184/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827442.421183:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827442.421185:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827442.421187:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011581184:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827442.548683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827442.548740:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827443.572691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827443.572780:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827444.596691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827444.596800:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827445.620699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827445.620749:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827446.644684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827446.644728:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827447.540710:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827447.540718:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827447.540722:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827447.540731:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2851 00000100:00080000:1.0:1787827447.540741:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827447.540746:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827447.540777:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011581312/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827447.540798:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827447.540801:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827447.540803:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827447.540805:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2851 00000100:00080000:1.0:1787827447.540808:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827447.540810:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827447.540818:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827447.540831:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011581440/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827447.540842:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011581312:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827447.540845:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827447.540853:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827447.540855:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827447.540860:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2851 00000100:00080000:1.0:1787827447.540871:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827447.540876:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827447.540880:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827447.540891:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011581568/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827447.540899:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827447.540901:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827447.540903:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827447.540905:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2851 00000100:00080000:1.0:1787827447.540908:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827447.540911:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827447.540913:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827447.540921:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011581696/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827447.540942:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011581440:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827447.540975:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011581568:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827447.540993:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011581696:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827447.668730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827447.668799:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827448.692777:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827448.692873:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827449.716741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827449.716814:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787827450.612889:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:0.0:1787827450.612938:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787827450.612955:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787827450.612996:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787827450.613003:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787827450.613011:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:2.0:1787827450.613053:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827447/real 1787827450] req@ffff907d908a1680 x1874670011581440/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827450.613084:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011581440/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827450.613099:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827450.613106:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827450.613112:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011581440:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827450.613122:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827447/real 1787827450] req@ffff907d908a3840 x1874670011581568/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827450.613135:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011581568/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827450.613144:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827450.613148:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827450.613153:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011581568:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827450.613161:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827447/real 1787827450] req@ffff907d908a12c0 x1874670011581696/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827450.613175:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011581696/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827450.613184:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827450.613188:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827450.613192:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011581696:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827450.613201:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827447/real 1787827450] req@ffff907d908a0f00 x1874670011581312/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827450.613216:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011581312/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827450.613224:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827450.613228:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827450.613231:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011581312:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827450.740746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827450.740809:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827451.764726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827451.764796:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827452.660696:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827452.660707:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827452.660711:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827452.660724:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2856 00000100:00080000:2.0:1787827452.660735:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827452.660742:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827452.660781:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011581824/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827452.660815:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827452.660818:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827452.660819:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827452.660822:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2856 00000100:00080000:2.0:1787827452.660825:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827452.660828:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827452.660838:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827452.660848:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011581952/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827452.660855:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011581824:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827452.660859:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827452.660864:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827452.660865:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827452.660869:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2856 00000100:00080000:2.0:1787827452.660871:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827452.660873:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827452.660877:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827452.660887:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011582080/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827452.660894:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827452.660897:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827452.660899:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827452.660904:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2856 00000100:00080000:2.0:1787827452.660922:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827452.660927:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827452.660934:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787827452.660935:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827452.660952:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011582208/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827452.660989:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827452/real 1787827452] req@ffff907d91149a40 x1874670011581824/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827452.661022:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011581824/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827452.661035:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827452.661040:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827452.661044:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011581824:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827452.661077:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011581952:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827452.661106:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827452.661122:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827452/real 1787827452] req@ffff907d9114a580 x1874670011581952/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827452.661135:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011581952/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827452.661145:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827452.661149:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827452.661153:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011581952:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827452.661177:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011582080:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827452.661203:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827452.661218:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827452/real 1787827452] req@ffff907d9114b840 x1874670011582080/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827452.661231:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011582080/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827452.661240:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827452.661244:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827452.661247:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011582080:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827452.661254:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011582208:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827452.661296:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827452.661312:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827452/real 1787827452] req@ffff907d9114ad00 x1874670011582208/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827452.661324:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011582208/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827452.661333:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827452.661337:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827452.661340:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011582208:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827452.788743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827452.788843:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827453.812728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827453.812814:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827454.836729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827454.836816:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827455.860682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827455.860781:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827456.884721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827456.884785:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827457.780780:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827457.780796:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827457.780801:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827457.780813:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2861 00000100:00080000:1.0:1787827457.780826:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827457.780833:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827457.780884:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011582336/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827457.780912:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827457.780915:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827457.780917:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827457.780920:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2861 00000100:00080000:1.0:1787827457.780922:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827457.780925:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827457.780935:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827457.780944:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011582464/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827457.780951:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827457.780954:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827457.780956:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827457.780959:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2861 00000100:00080000:1.0:1787827457.780961:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827457.780963:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827457.780967:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827457.780976:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011582336:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827457.780979:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011582592/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827457.780995:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827457.780999:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827457.781001:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827457.781005:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2861 00000100:00080000:1.0:1787827457.781010:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827457.781015:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827457.781021:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827457.781034:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011582720/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827457.781073:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827457.781139:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827457/real 1787827457] req@ffff907d908a21c0 x1874670011582336/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827457.781163:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011582336/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827457.781178:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827457.781183:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827457.781188:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011582336:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827457.781197:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011582464:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827457.781225:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827457.781256:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827457/real 1787827457] req@ffff907d908a2580 x1874670011582464/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827457.781272:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011582464/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827457.781283:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827457.781287:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827457.781291:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011582464:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827457.781320:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011582592:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827457.781346:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827457.781359:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827457/real 1787827457] req@ffff907d908a2940 x1874670011582592/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827457.781375:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011582592/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827457.781384:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827457.781387:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827457.781391:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011582592:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827457.781398:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011582720:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827457.781420:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827457.781432:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827457/real 1787827457] req@ffff907d908a0000 x1874670011582720/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827457.781445:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011582720/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827457.781455:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827457.781460:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827457.781464:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011582720:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827457.908688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827457.908781:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827458.932742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827458.932826:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827459.956692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827459.956784:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827460.980748:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827460.980812:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827462.004678:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827462.004722:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827462.900690:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827462.900701:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827462.900705:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827462.900716:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2866 00000100:00080000:1.0:1787827462.900727:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827462.900734:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827462.900775:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011582848/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827462.900802:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827462.900805:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827462.900806:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827462.900809:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2866 00000100:00080000:1.0:1787827462.900812:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827462.900814:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827462.900823:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827462.900842:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011582976/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827462.900852:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011582848:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827462.900859:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827462.900867:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827462.900869:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827462.900875:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2866 00000100:00080000:1.0:1787827462.900880:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827462.900885:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827462.900891:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827462.900905:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011583104/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827462.900913:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827462.900916:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827462.900918:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827462.900921:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2866 00000800:00000100:3.0:1787827462.900926:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787827462.900929:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827462.900936:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827462.900943:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827462.900957:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011583232/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827462.900983:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827462/real 1787827462] req@ffff907d908a2d00 x1874670011582848/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827462.901008:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011582848/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827462.901022:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827462.901027:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827462.901032:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011582848:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827462.901059:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011582976:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827462.901080:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827462.901090:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827462/real 1787827462] req@ffff907d908a1e00 x1874670011582976/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827462.901098:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011582976/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827462.901104:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827462.901107:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827462.901109:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011582976:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827462.901113:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011583104:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827462.901125:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827462.901131:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827462/real 1787827462] req@ffff907d908a0000 x1874670011583104/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827462.901139:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011583104/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827462.901148:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827462.901150:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827462.901152:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011583104:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827462.901155:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011583232:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827462.901165:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827462.901170:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827462/real 1787827462] req@ffff907d908a2940 x1874670011583232/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827462.901177:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011583232/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827462.901183:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827462.901185:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827462.901187:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011583232:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827463.028692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827463.028784:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827464.052674:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827464.052728:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827465.076712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827465.076772:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827466.100692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827466.100778:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827467.124697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827467.124747:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827468.020713:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827468.020721:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827468.020725:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827468.020734:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2871 00000100:00080000:2.0:1787827468.020744:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827468.020750:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827468.020781:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011583360/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827468.020801:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827468.020804:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827468.020805:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827468.020808:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2871 00000100:00080000:2.0:1787827468.020811:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827468.020813:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827468.020822:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827468.020831:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011583488/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827468.020838:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827468.020849:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1787827468.020850:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011583360:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827468.020851:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827468.020857:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2871 00000100:00080000:2.0:1787827468.020861:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827468.020864:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827468.020867:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827468.020877:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011583616/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827468.020883:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827468.020890:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827468.020892:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827468.020895:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2871 00000100:00080000:2.0:1787827468.020907:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827468.020910:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827468.020914:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787827468.020918:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827468.020925:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011583744/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827468.020957:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827468/real 1787827468] req@ffff907d91149680 x1874670011583360/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827468.020972:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011583360/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827468.020981:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827468.020984:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827468.020987:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011583360:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827468.021004:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011583488:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827468.021019:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827468.021027:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827468/real 1787827468] req@ffff907d9114bc00 x1874670011583488/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827468.021035:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011583488/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827468.021040:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827468.021042:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827468.021044:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011583488:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827468.021048:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011583616:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827468.021058:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827468.021064:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827468/real 1787827468] req@ffff907d911483c0 x1874670011583616/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827468.021071:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011583616/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827468.021076:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827468.021078:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827468.021080:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011583616:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827468.021084:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011583744:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827468.021093:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827468.021098:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827468/real 1787827468] req@ffff907d9114b480 x1874670011583744/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827468.021105:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011583744/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827468.021110:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827468.021112:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827468.021114:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011583744:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827468.148700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827468.148747:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827469.172729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827469.172802:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827470.196715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827470.196798:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827471.220727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827471.220815:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827472.244703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827472.244882:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827473.140767:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827473.140778:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827473.140783:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827473.140793:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2876 00000100:00080000:1.0:1787827473.140805:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827473.140813:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827473.140849:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011583872/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827473.140880:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827473.140884:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827473.140886:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827473.140890:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2876 00000100:00080000:1.0:1787827473.140893:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827473.140898:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827473.140908:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827473.140926:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011584000/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827473.140926:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011583872:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827473.140942:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827473.140946:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827473.140949:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827473.140952:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2876 00000100:00080000:1.0:1787827473.140956:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827473.140960:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827473.140965:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827473.140977:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011584128/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827473.140988:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787827473.140989:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827473.140994:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827473.140997:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827473.141002:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2876 00000100:00080000:1.0:1787827473.141007:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827473.141011:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827473.141016:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827473.141027:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011584256/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827473.141033:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827473/real 1787827473] req@ffff907d908a2580 x1874670011583872/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827473.141056:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011583872/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827473.141064:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827473.141068:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827473.141071:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011583872:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827473.141090:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011584000:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827473.141105:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827473.141114:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827473/real 1787827473] req@ffff907d908a21c0 x1874670011584000/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827473.141122:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011584000/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827473.141128:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827473.141130:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827473.141145:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011584000:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827473.141150:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011584128:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827473.141166:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827473.141174:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827473/real 1787827473] req@ffff907d908a0f00 x1874670011584128/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827473.141181:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011584128/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827473.141190:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827473.141193:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827473.141196:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011584128:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827473.141202:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011584256:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827473.141221:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827473.141233:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827473/real 1787827473] req@ffff907d908a12c0 x1874670011584256/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827473.141245:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011584256/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827473.141251:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827473.141253:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827473.141255:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011584256:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827473.268702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827473.268755:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827474.292746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827474.292811:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827475.316733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827475.316749:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827476.340732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827476.340889:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827477.364707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827477.364761:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827478.260725:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827478.260737:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827478.260743:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827478.260756:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2882 00000100:00080000:3.0:1787827478.260770:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827478.260778:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827478.260826:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011584384/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827478.260859:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827478.260864:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827478.260866:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827478.260872:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2882 00000100:00080000:3.0:1787827478.260877:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827478.260884:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827478.260896:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827478.260902:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011584384:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827478.260919:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011584512/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827478.260932:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827478.260936:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827478.260939:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827478.260944:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2882 00000100:00080000:3.0:1787827478.260948:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827478.260954:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827478.260961:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827478.260974:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011584640/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827478.260986:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787827478.260987:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787827478.260994:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827478.260997:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827478.261002:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2882 00000100:00100000:1.0:1787827478.261036:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827478/real 1787827478] req@ffff907da562c3c0 x1874670011584384/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827478.261053:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011584384/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827478.261062:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827478.261065:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827478.261068:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011584384:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827478.261089:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787827478.261094:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011584512:192.168.203.104@tcp:38:kworker.0 00000100:00080000:3.0:1787827478.261113:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:1.0:1787827478.261134:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827478.261145:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827478/real 1787827478] req@ffff907da562f0c0 x1874670011584512/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827478.261154:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011584512/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827478.261160:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827478.261163:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827478.261166:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011584512:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827478.261170:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011584640:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827478.261184:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827478.261190:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827478/real 1787827478] req@ffff907da562fc00 x1874670011584640/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827478.261198:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011584640/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827478.261204:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827478.261207:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827478.261209:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011584640:192.168.203.104@tcp:8:kworker.0 00000008:00100000:3.0:1787827478.261258:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827478.261324:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011584768/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827478.261356:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011584768:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827478.261382:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827478.261394:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827478/real 1787827478] req@ffff907da562e940 x1874670011584768/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827478.261404:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011584768/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827478.261411:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827478.261414:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827478.261416:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011584768:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827478.388730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827478.388798:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827479.412696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827479.412753:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827480.436766:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827480.436783:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827481.460718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827481.460837:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827482.484711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827482.484807:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827483.380700:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827483.380709:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827483.380713:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827483.380721:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2887 00000100:00080000:3.0:1787827483.380731:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827483.380737:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827483.380770:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011584896/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827483.380806:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827483.380811:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827483.380813:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827483.380820:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2887 00000100:00080000:3.0:1787827483.380825:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827483.380831:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827483.380842:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827483.380843:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011584896:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827483.380863:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011585024/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827483.380875:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827483.380878:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827483.380879:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827483.380883:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2887 00000100:00080000:3.0:1787827483.380904:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827483.380908:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827483.380913:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827483.380924:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011585152/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827483.380932:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787827483.380940:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827483.380943:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827483.380945:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827483.380949:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2887 00000100:00080000:3.0:1787827483.380955:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827483.380957:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827483.380961:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827483.380970:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011585280/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827483.380994:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827483/real 1787827483] req@ffff907da562cb40 x1874670011584896/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827483.381013:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011584896/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827483.381027:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827483.381033:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827483.381038:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011584896:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827483.381094:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011585024:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827483.381124:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827483.381138:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827483/real 1787827483] req@ffff907da562da40 x1874670011585024/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827483.381153:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011585024/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827483.381163:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827483.381167:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827483.381171:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011585024:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827483.381178:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011585152:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827483.381203:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827483.381218:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827483/real 1787827483] req@ffff907da562f840 x1874670011585152/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827483.381239:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011585152/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827483.381254:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827483.381257:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827483.381259:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011585152:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827483.381264:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011585280:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827483.381282:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827483.381290:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827483/real 1787827483] req@ffff907da562e1c0 x1874670011585280/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827483.381299:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011585280/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827483.381305:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827483.381307:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827483.381309:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011585280:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827483.508665:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827483.508763:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827484.532702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827484.532757:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827485.556667:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827485.556780:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827486.580668:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827486.580722:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827487.604792:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827487.604918:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827488.500815:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827488.500825:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827488.500830:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827488.500840:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2892 00000100:00080000:2.0:1787827488.500850:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827488.500858:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827488.500916:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011585408/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827488.500967:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827488.500971:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827488.500974:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827488.500979:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2892 00000100:00080000:2.0:1787827488.500983:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827488.500987:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827488.501000:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827488.501018:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011585536/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827488.501075:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827488.501082:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787827488.501084:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011585408:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827488.501084:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827488.501102:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2892 00000100:00080000:2.0:1787827488.501106:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827488.501111:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827488.501117:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827488.501151:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011585664/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827488.501161:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827488.501165:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827488.501167:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827488.501171:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2892 00000100:00080000:2.0:1787827488.501176:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827488.501180:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827488.501185:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827488.501204:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011585792/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827488.501206:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827488.501266:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827488/real 1787827488] req@ffff907d9114b480 x1874670011585408/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827488.501286:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011585408/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827488.501302:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827488.501306:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827488.501310:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011585408:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827488.501335:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011585536:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827488.501353:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827488.501363:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827488/real 1787827488] req@ffff907d911483c0 x1874670011585536/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827488.501371:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011585536/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827488.501377:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827488.501380:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827488.501382:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011585536:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827488.501386:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011585664:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827488.501398:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827488.501404:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827488/real 1787827488] req@ffff907d9114bc00 x1874670011585664/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827488.501412:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011585664/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827488.501422:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827488.501426:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827488.501429:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011585664:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827488.501437:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011585792:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827488.501470:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827488.501502:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827488/real 1787827488] req@ffff907d91149680 x1874670011585792/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827488.501519:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011585792/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827488.501530:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827488.501535:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827488.501539:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011585792:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827488.628722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827488.628832:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827489.652698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827489.652772:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827490.676817:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827490.676823:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827491.700700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827491.700760:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827492.724672:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827492.724736:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827493.620704:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827493.620718:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827493.620724:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827493.620737:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2897 00000100:00080000:2.0:1787827493.620751:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827493.620760:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827493.620808:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011585920/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827493.620858:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827493.620863:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827493.620866:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827493.620871:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2897 00000100:00080000:2.0:1787827493.620875:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827493.620880:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827493.620893:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827493.620910:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011586048/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827493.620925:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827493.620929:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827493.620932:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827493.620939:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2897 00000100:00080000:2.0:1787827493.620943:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827493.620948:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827493.620954:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827493.620966:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011586176/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827493.620980:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787827493.621008:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011585920:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827493.621013:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827493.621023:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827493.621048:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2897 00000100:00080000:2.0:1787827493.621054:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827493.621059:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827493.621069:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827493.621094:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011586304/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827493.621160:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827493.621254:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827493/real 1787827493] req@ffff907d9114ad00 x1874670011585920/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827493.621286:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011585920/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827493.621306:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827493.621331:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827493.621340:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011585920:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827493.621357:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011586048:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827493.621398:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827493.621434:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827493/real 1787827493] req@ffff907d9114b840 x1874670011586048/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827493.621456:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011586048/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827493.621465:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827493.621468:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827493.621471:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011586048:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827493.621502:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011586176:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827493.621523:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827493.621532:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827493/real 1787827493] req@ffff907d9114a580 x1874670011586176/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827493.621542:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011586176/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827493.621549:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827493.621552:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827493.621555:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011586176:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827493.621560:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011586304:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827493.621583:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827493.621591:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827493/real 1787827493] req@ffff907d91149a40 x1874670011586304/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827493.621601:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011586304/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827493.621647:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827493.621650:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827493.621653:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011586304:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827493.748712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827493.748793:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827494.772692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827494.772720:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827495.796697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827495.796755:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827496.820678:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827496.820716:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827497.844749:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827497.844858:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827498.740773:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827498.740787:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827498.740792:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827498.740805:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2902 00000100:00080000:3.0:1787827498.740820:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827498.740828:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827498.740875:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011586432/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827498.740910:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827498.740914:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827498.740916:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827498.740926:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2902 00000100:00080000:3.0:1787827498.740930:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827498.740935:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827498.740947:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827498.740979:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011586560/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827498.740993:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827498.741008:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827498.741012:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787827498.741013:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011586432:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827498.741020:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2902 00000100:00080000:3.0:1787827498.741026:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827498.741031:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827498.741038:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827498.741053:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011586688/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827498.741063:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827498.741068:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827498.741070:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827498.741075:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2902 00000100:00080000:3.0:1787827498.741079:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827498.741083:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827498.741089:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827498.741101:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011586816/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827498.741121:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827498.741187:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827498/real 1787827498] req@ffff907da562de00 x1874670011586432/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827498.741222:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011586432/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827498.741235:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827498.741240:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827498.741244:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011586432:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827498.741274:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011586560:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827498.741303:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827498.741318:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827498/real 1787827498] req@ffff907da562e1c0 x1874670011586560/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827498.741333:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011586560/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827498.741343:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827498.741347:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827498.741351:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011586560:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827498.741359:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011586688:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827498.741383:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827498.741397:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827498/real 1787827498] req@ffff907da562f840 x1874670011586688/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827498.741411:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011586688/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827498.741421:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827498.741424:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827498.741428:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011586688:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827498.741435:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011586816:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827498.741458:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827498.741472:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827498/real 1787827498] req@ffff907da562da40 x1874670011586816/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827498.741484:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011586816/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827498.741493:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827498.741497:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827498.741501:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011586816:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827498.868679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827498.868734:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827499.892724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827499.892880:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787827500.917665:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827500.917780:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787827501.940795:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827501.940812:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787827502.964728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827502.964862:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827503.860737:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827503.860745:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827503.860749:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827503.860759:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2907 00000100:00080000:3.0:1787827503.860771:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827503.860780:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827503.860831:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011586944/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827503.860857:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827503.860861:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827503.860862:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827503.860868:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2907 00000100:00080000:3.0:1787827503.860871:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827503.860874:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827503.860882:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827503.860893:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011587072/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827503.860900:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827503.860903:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827503.860905:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827503.860907:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2907 00000100:00080000:3.0:1787827503.860910:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787827503.860915:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011586944:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827503.860917:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827503.860937:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827503.860952:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011587200/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827503.860964:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827503.860968:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827503.860971:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827503.860974:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2907 00000100:00080000:3.0:1787827503.860979:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827503.860983:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827503.860989:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827503.861002:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011587328/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827503.861003:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827503.861061:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827503/real 1787827503] req@ffff907da562cb40 x1874670011586944/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827503.861080:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011586944/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827503.861102:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827503.861107:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827503.861112:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011586944:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827503.861121:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011587072:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827503.861146:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827503.861160:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827503/real 1787827503] req@ffff907da562e940 x1874670011587072/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827503.861174:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011587072/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827503.861184:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827503.861187:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827503.861191:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011587072:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827503.861216:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011587200:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827503.861239:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827503.861257:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827503/real 1787827503] req@ffff907da562fc00 x1874670011587200/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827503.861271:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011587200/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827503.861287:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827503.861290:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827503.861294:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011587200:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827503.861300:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011587328:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827503.861322:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827503.861334:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827503/real 1787827503] req@ffff907da562f0c0 x1874670011587328/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827503.861346:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011587328/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827503.861355:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827503.861358:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827503.861362:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011587328:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827503.988735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827503.988776:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827505.012708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827505.012764:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827506.036714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827506.036783:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827507.060702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827507.060780:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827508.084708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827508.084816:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827508.980750:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827508.980760:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827508.980764:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827508.980785:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2912 00000100:00080000:2.0:1787827508.980796:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827508.980804:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827508.980840:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011587456/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827508.980871:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827508.980875:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827508.980877:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827508.980882:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2912 00000100:00080000:2.0:1787827508.980899:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827508.980903:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827508.980913:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827508.980930:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011587584/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827508.980935:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011587456:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827508.980950:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827508.980954:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827508.980956:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827508.980961:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2912 00000100:00080000:2.0:1787827508.980965:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827508.980969:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827508.980975:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827508.980989:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011587712/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827508.980999:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827508.981002:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827508.981004:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827508.981011:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2912 00000800:00000100:1.0:1787827508.981012:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787827508.981018:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827508.981024:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827508.981030:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827508.981043:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011587840/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827508.981059:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827508/real 1787827508] req@ffff907d9114a940 x1874670011587456/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827508.981091:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011587456/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827508.981102:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827508.981106:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827508.981110:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011587456:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827508.981132:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011587584:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827508.981156:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827508.981169:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827508/real 1787827508] req@ffff907d911492c0 x1874670011587584/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827508.981183:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011587584/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827508.981203:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827508.981207:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827508.981210:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011587584:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827508.981217:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011587712:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827508.981240:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827508.981253:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827508/real 1787827508] req@ffff907d91148f00 x1874670011587712/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827508.981266:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011587712/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827508.981274:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827508.981278:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827508.981281:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011587712:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827508.981288:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011587840:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827508.981307:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827508.981317:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827508/real 1787827508] req@ffff907d91148b40 x1874670011587840/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827508.981329:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011587840/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827508.981337:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827508.981341:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827508.981344:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011587840:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827509.108820:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827509.108820:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827510.132711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827510.132792:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827511.156743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827511.156835:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827512.180701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827512.180773:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827513.204759:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827513.204834:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827514.100763:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827514.100790:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827514.100796:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827514.100808:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2917 00000100:00080000:0.0:1787827514.100825:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827514.100863:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827514.100901:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011587968/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827514.100926:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827514.100929:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827514.100933:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827514.100938:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2917 00000100:00080000:0.0:1787827514.100941:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827514.100945:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827514.100955:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827514.100968:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011588096/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827514.100981:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827514.100984:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787827514.100988:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787827514.100992:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011587968:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787827514.100994:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2917 00000100:00080000:0.0:1787827514.101011:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827514.101016:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827514.101020:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827514.101037:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011588224/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827514.101047:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827514.101050:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827514.101053:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827514.101057:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2917 00000100:00080000:0.0:1787827514.101072:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827514.101078:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827514.101084:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827514.101100:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011588352/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827514.101137:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011588096:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827514.101199:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011588224:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827514.101245:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011588352:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827514.228707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827514.228764:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827515.252706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827515.252801:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827516.276703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827516.276829:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787827517.172854:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:0.0:1787827517.172898:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787827517.172915:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787827517.172955:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787827517.172973:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787827517.172986:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:1.0:1787827517.173018:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827514/real 1787827517] req@ffff907d90847c00 x1874670011588096/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827517.173051:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011588096/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827517.173062:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827517.173068:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827517.173073:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011588096:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827517.173081:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827514/real 1787827517] req@ffff907d90846580 x1874670011588224/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827517.173102:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011588224/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827517.173110:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827517.173114:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827517.173118:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011588224:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827517.173125:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827514/real 1787827517] req@ffff907d90845e00 x1874670011588352/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827517.173137:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011588352/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827517.173146:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827517.173149:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827517.173153:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011588352:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827517.173174:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827514/real 1787827517] req@ffff907d908470c0 x1874670011587968/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827517.173190:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011587968/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827517.173198:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827517.173202:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827517.173205:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011587968:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827517.300737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827517.300798:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827518.324690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827518.324731:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827519.220705:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827519.220714:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827519.220717:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827519.220726:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2922 00000100:00080000:0.0:1787827519.220735:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827519.220741:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827519.220769:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011588480/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827519.220794:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827519.220796:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827519.220798:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827519.220801:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2922 00000100:00080000:0.0:1787827519.220803:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827519.220806:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827519.220826:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827519.220839:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011588608/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827519.220839:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011588480:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787827519.220854:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827519.220856:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787827519.220858:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827519.220861:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2922 00000100:00080000:0.0:1787827519.220864:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827519.220866:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827519.220872:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827519.220885:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011588736/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827519.220896:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827519.220900:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827519.220902:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827519.220906:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2922 00000800:00000100:1.0:1787827519.220913:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787827519.220914:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827519.220922:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827519.220929:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827519.220942:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011588864/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827519.220960:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827519/real 1787827519] req@ffff907d90844b40 x1874670011588480/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827519.220984:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011588480/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827519.220997:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827519.221001:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827519.221005:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011588480:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827519.221028:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011588608:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827519.221055:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827519.221069:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827519/real 1787827519] req@ffff907d908443c0 x1874670011588608/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827519.221081:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011588608/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827519.221090:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827519.221094:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827519.221097:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011588608:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827519.221103:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011588736:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827519.221125:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827519.221137:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827519/real 1787827519] req@ffff907d90844000 x1874670011588736/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827519.221149:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011588736/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827519.221158:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827519.221161:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827519.221165:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011588736:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827519.221171:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011588864:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827519.221192:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827519.221204:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827519/real 1787827519] req@ffff907d90844780 x1874670011588864/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827519.221217:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011588864/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827519.221225:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827519.221229:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827519.221232:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011588864:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827519.348741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827519.348809:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827520.372723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827520.372818:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827521.396709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827521.396752:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827522.420733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827522.420753:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827523.444743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827523.444829:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827524.340698:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827524.340706:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827524.340710:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827524.340719:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2928 00000100:00080000:3.0:1787827524.340728:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827524.340745:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827524.340793:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011588992/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827524.340831:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827524.340834:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827524.340836:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827524.340841:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2928 00000100:00080000:3.0:1787827524.340845:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827524.340850:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827524.340859:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827524.340878:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011588992:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827524.340879:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011589120/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827524.340896:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827524.340900:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827524.340903:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827524.340906:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2928 00000100:00080000:3.0:1787827524.340910:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827524.340913:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827524.340917:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827524.340926:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011589248/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827524.340934:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827524.340937:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827524.340938:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827524.340940:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2928 00000100:00080000:3.0:1787827524.340942:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827524.340944:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827524.340948:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787827524.340952:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827524.340958:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011589376/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827524.340992:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827524/real 1787827524] req@ffff907da562c3c0 x1874670011588992/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827524.341006:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011588992/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827524.341014:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827524.341017:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827524.341020:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011588992:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827524.341037:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011589120:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827524.341054:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827524.341063:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827524/real 1787827524] req@ffff907da562d2c0 x1874670011589120/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827524.341070:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011589120/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827524.341076:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827524.341078:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827524.341081:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011589120:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827524.341085:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011589248:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827524.341096:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827524.341102:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827524/real 1787827524] req@ffff907da562e580 x1874670011589248/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827524.341109:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011589248/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827524.341114:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827524.341115:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827524.341117:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011589248:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827524.341121:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011589376:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827524.341138:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827524.341149:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827524/real 1787827524] req@ffff907da562c000 x1874670011589376/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827524.341162:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011589376/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827524.341171:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827524.341175:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827524.341186:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011589376:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827524.468668:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827524.468712:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827525.492809:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827525.492811:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787827526.516732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827526.516827:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827527.540747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827527.540769:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827528.564731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827528.564827:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827529.460829:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827529.460841:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827529.460846:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827529.460858:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2933 00000100:00080000:0.0:1787827529.460871:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827529.460879:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827529.460933:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011589504/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827529.460974:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827529.460976:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827529.460978:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827529.460981:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2933 00000100:00080000:0.0:1787827529.460984:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827529.460987:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827529.460995:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827529.461005:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011589632/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827529.461014:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787827529.461014:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011589504:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787827529.461025:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787827529.461027:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827529.461030:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2933 00000100:00080000:0.0:1787827529.461032:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827529.461034:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827529.461038:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827529.461045:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011589760/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827529.461052:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827529.461055:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827529.461056:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827529.461059:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2933 00000100:00080000:0.0:1787827529.461068:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827529.461071:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827529.461074:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787827529.461080:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827529.461084:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011589888/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827529.461119:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827529/real 1787827529] req@ffff907d90845680 x1874670011589504/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827529.461134:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011589504/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827529.461141:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827529.461144:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827529.461147:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011589504:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827529.461163:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011589632:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827529.461180:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827529.461192:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827529/real 1787827529] req@ffff907d90846d00 x1874670011589632/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827529.461205:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011589632/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827529.461214:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827529.461218:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827529.461221:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011589632:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827529.461228:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011589760:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827529.461252:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827529.461265:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827529/real 1787827529] req@ffff907d908452c0 x1874670011589760/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827529.461278:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011589760/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827529.461299:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827529.461302:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827529.461306:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011589760:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827529.461312:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011589888:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827529.461327:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827529.461335:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827529/real 1787827529] req@ffff907d90846940 x1874670011589888/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827529.461342:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011589888/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827529.461348:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827529.461350:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827529.461352:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011589888:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827529.588758:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827529.588861:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827530.612759:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827530.612828:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827531.636814:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827531.636841:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787827532.660750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827532.660824:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827533.684765:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827533.684818:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827534.580764:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827534.580774:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827534.580778:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827534.580787:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2938 00000100:00080000:3.0:1787827534.580809:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827534.580816:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827534.580851:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011590016/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827534.580888:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827534.580892:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827534.580893:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827534.580897:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2938 00000100:00080000:3.0:1787827534.580900:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827534.580903:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827534.580912:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827534.580923:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011590144/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827534.580932:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827534.580935:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827534.580937:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827534.580939:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2938 00000100:00080000:3.0:1787827534.580942:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827534.580947:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827534.580951:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011590016:192.168.203.104@tcp:250:kworker.0 00000008:00100000:3.0:1787827534.580964:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827534.580975:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011590272/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827534.580983:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827534.580986:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827534.580987:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827534.580990:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2938 00000100:00080000:3.0:1787827534.580994:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827534.580996:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827534.581000:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827534.581011:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011590400/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827534.581044:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827534.581108:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827534/real 1787827534] req@ffff907da562f480 x1874670011590016/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827534.581124:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011590016/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827534.581133:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827534.581136:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827534.581140:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011590016:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827534.581146:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011590144:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827534.581164:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827534.581173:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827534/real 1787827534] req@ffff907da562c780 x1874670011590144/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827534.581182:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011590144/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827534.581188:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827534.581191:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827534.581193:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011590144:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827534.581222:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011590272:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827534.581235:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827534.581242:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827534/real 1787827534] req@ffff907da562ed00 x1874670011590272/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827534.581256:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011590272/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827534.581262:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827534.581264:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827534.581266:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011590272:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827534.581270:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011590400:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827534.581281:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827534.581288:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827534/real 1787827534] req@ffff907da562cf00 x1874670011590400/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827534.581296:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011590400/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827534.581302:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827534.581304:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827534.581306:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011590400:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827534.708740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827534.708768:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827535.732757:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827535.732777:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787827536.756827:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827536.756872:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827537.780725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827537.780759:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827538.804798:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827538.804893:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827539.700861:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827539.700893:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827539.700900:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827539.700917:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2943 00000100:00080000:0.0:1787827539.700935:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827539.700945:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827539.701009:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011590528/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827539.701066:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827539.701071:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827539.701074:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827539.701079:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2943 00000100:00080000:0.0:1787827539.701084:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827539.701088:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827539.701103:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827539.701121:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011590656/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827539.701135:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827539.701140:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787827539.701143:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827539.701149:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2943 00000100:00080000:0.0:1787827539.701153:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827539.701160:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827539.701173:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827539.701175:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011590528:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827539.701191:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011590784/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827539.701205:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827539.701210:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827539.701213:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827539.701219:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2943 00000100:00080000:0.0:1787827539.701224:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827539.701229:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827539.701235:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827539.701254:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011590912/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827539.701286:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827539.701344:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827539/real 1787827539] req@ffff907d90844f00 x1874670011590528/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827539.701379:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011590528/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827539.701392:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827539.701397:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827539.701402:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011590528:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827539.701412:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011590656:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827539.701439:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827539.701455:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827539/real 1787827539] req@ffff907d90845a40 x1874670011590656/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827539.701477:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011590656/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827539.701492:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827539.701498:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827539.701504:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011590656:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827539.701555:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011590784:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827539.701584:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827539.701603:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827539/real 1787827539] req@ffff907d90847480 x1874670011590784/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827539.701719:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011590784/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827539.701737:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827539.701741:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827539.701744:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011590784:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827539.701752:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011590912:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827539.701779:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827539.701792:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827539/real 1787827539] req@ffff907d90846940 x1874670011590912/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827539.701804:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011590912/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827539.701813:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827539.701816:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827539.701820:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011590912:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827539.828810:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827539.828959:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827540.852725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827540.852781:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827541.876777:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827541.876914:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827542.900847:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827542.900953:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827543.924719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827543.924822:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827544.820759:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827544.820770:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827544.820775:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827544.820786:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2948 00000100:00080000:2.0:1787827544.820798:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827544.820805:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827544.820849:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011591040/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827544.820877:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827544.820880:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827544.820882:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827544.820885:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2948 00000100:00080000:2.0:1787827544.820887:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827544.820890:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827544.820899:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827544.820911:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011591168/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827544.820918:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827544.820921:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827544.820922:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827544.820925:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2948 00000100:00080000:2.0:1787827544.820927:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827544.820929:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827544.820933:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827544.820941:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011591296/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827544.820947:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827544.820950:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827544.820952:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827544.820956:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2948 00000100:00080000:2.0:1787827544.820958:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827544.820961:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827544.820963:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827544.820976:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011591424/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827544.820980:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011591040:192.168.203.104@tcp:250:kworker.0 00000800:00000100:1.0:1787827544.821054:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827544.821091:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827544/real 1787827544] req@ffff907d91148b40 x1874670011591040/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827544.821110:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011591040/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827544.821118:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827544.821122:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827544.821125:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011591040:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827544.821130:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011591168:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827544.821145:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827544.821153:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827544/real 1787827544] req@ffff907d91148f00 x1874670011591168/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827544.821161:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011591168/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827544.821166:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827544.821168:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827544.821170:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011591168:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827544.821174:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011591296:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827544.821185:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827544.821191:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827544/real 1787827544] req@ffff907d911492c0 x1874670011591296/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827544.821198:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011591296/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827544.821207:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827544.821210:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827544.821212:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011591296:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827544.821232:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011591424:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827544.821242:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827544.821248:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827544/real 1787827544] req@ffff907d9114a940 x1874670011591424/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827544.821255:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011591424/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827544.821269:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827544.821272:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827544.821275:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011591424:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827544.948725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827544.948782:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827545.972691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827545.972731:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827546.996755:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827546.996806:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827548.020687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827548.020771:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827549.044718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827549.044753:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827549.940670:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827549.940679:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827549.940684:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827549.940709:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2953 00000100:00080000:2.0:1787827549.940718:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827549.940726:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827549.940760:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011591552/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827549.940789:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827549.940793:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827549.940795:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827549.940800:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2953 00000100:00080000:2.0:1787827549.940808:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827549.940813:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827549.940822:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827549.940835:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011591552:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827549.940859:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011591680/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827549.940872:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827549.940875:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827549.940877:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827549.940880:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2953 00000100:00080000:2.0:1787827549.940883:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827549.940886:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827549.940891:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827549.940904:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011591808/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827549.940913:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827549.940917:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827549.940918:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827549.940923:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2953 00000800:00000100:0.0:1787827549.940924:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787827549.940927:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827549.940947:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827549.940955:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827549.940969:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011591936/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827549.941011:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827549/real 1787827549] req@ffff907d91149a40 x1874670011591552/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827549.941027:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011591552/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827549.941039:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827549.941042:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827549.941046:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011591552:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827549.941064:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011591680:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827549.941081:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827549.941089:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827549/real 1787827549] req@ffff907d9114a580 x1874670011591680/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827549.941097:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011591680/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827549.941103:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827549.941105:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827549.941108:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011591680:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827549.941113:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011591808:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827549.941127:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827549.941133:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827549/real 1787827549] req@ffff907d9114b840 x1874670011591808/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827549.941141:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011591808/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827549.941146:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827549.941148:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827549.941150:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011591808:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827549.941153:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011591936:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827549.941165:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827549.941171:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827549/real 1787827549] req@ffff907d9114ad00 x1874670011591936/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827549.941179:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011591936/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827549.941185:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827549.941187:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827549.941190:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011591936:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827550.068793:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827550.068838:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827551.092756:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827551.092815:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827552.116741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827552.116834:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827553.140787:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827553.140800:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827554.164715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827554.164785:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827555.060742:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827555.060752:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827555.060757:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827555.060768:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2958 00000100:00080000:2.0:1787827555.060780:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827555.060787:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827555.060825:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011592064/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827555.060853:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827555.060856:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827555.060858:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827555.060862:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2958 00000100:00080000:2.0:1787827555.060865:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827555.060868:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827555.060877:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827555.060887:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011592192/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827555.060896:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827555.060899:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827555.060900:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827555.060903:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2958 00000100:00080000:2.0:1787827555.060905:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827555.060919:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827555.060925:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827555.060934:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011592320/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827555.060945:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787827555.060950:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011592064:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827555.060950:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827555.060954:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827555.060964:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2958 00000100:00080000:2.0:1787827555.060967:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827555.060970:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827555.060974:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827555.060986:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011592448/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827555.061043:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827555.061078:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827555/real 1787827555] req@ffff907d91149680 x1874670011592064/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827555.061092:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011592064/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827555.061101:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827555.061104:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827555.061107:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011592064:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827555.061112:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011592192:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827555.061127:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827555.061135:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827555/real 1787827555] req@ffff907d9114bc00 x1874670011592192/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827555.061144:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011592192/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827555.061150:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827555.061152:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827555.061154:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011592192:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827555.061172:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011592320:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827555.061191:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827555.061203:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827555/real 1787827555] req@ffff907d911483c0 x1874670011592320/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827555.061223:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011592320/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827555.061233:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827555.061236:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827555.061240:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011592320:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827555.061247:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011592448:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827555.061265:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827555.061273:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827555/real 1787827555] req@ffff907d9114b480 x1874670011592448/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827555.061282:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011592448/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827555.061288:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827555.061290:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827555.061292:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011592448:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827555.188727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827555.188812:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827556.212769:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827556.212875:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827557.236674:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827557.236769:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827558.260713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827558.260798:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827559.284689:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827559.284778:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827560.180695:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827560.180704:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827560.180707:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827560.180716:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2963 00000100:00080000:1.0:1787827560.180726:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827560.180734:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827560.180771:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011592576/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827560.180797:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827560.180800:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827560.180801:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827560.180813:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2963 00000100:00080000:1.0:1787827560.180818:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827560.180824:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827560.180835:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827560.180841:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011592576:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827560.180858:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011592704/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827560.180871:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827560.180875:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827560.180877:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827560.180881:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2963 00000100:00080000:1.0:1787827560.180892:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827560.180896:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827560.180902:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787827560.180905:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827560.180919:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011592832/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827560.180941:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827560.180945:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827560.180947:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787827560.180949:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827560/real 1787827560] req@ffff907d908a3840 x1874670011592576/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827560.180954:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2963 00000100:00080000:1.0:1787827560.180958:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827560.180964:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827560.180974:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827560.180984:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011592576/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827560.180991:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011592960/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827560.181020:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827560.181025:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827560.181029:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011592576:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827560.181048:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011592704:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827560.181066:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827560.181075:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827560/real 1787827560] req@ffff907d908a1680 x1874670011592704/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827560.181083:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011592704/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827560.181088:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827560.181091:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827560.181093:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011592704:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827560.181097:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011592832:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827560.181108:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827560.181114:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827560/real 1787827560] req@ffff907d908a12c0 x1874670011592832/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827560.181121:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011592832/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827560.181126:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827560.181128:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827560.181130:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011592832:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827560.181134:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011592960:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827560.181143:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827560.181149:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827560/real 1787827560] req@ffff907d908a0f00 x1874670011592960/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827560.181156:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011592960/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827560.181161:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827560.181163:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827560.181165:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011592960:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827560.308709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827560.308741:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827561.332663:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827561.332730:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827562.356705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827562.356750:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827563.380669:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827563.380771:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827564.404685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827564.404741:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827565.300708:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827565.300717:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827565.300721:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827565.300731:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2969 00000100:00080000:2.0:1787827565.300742:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827565.300762:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827565.300803:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011593088/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827565.300832:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827565.300835:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827565.300837:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827565.300840:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2969 00000100:00080000:2.0:1787827565.300844:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827565.300847:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827565.300856:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827565.300869:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011593216/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827565.300878:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827565.300882:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787827565.300884:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011593088:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827565.300885:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827565.300895:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2969 00000100:00080000:2.0:1787827565.300900:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827565.300904:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827565.300911:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827565.300927:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011593344/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827565.300939:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827565.300944:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827565.300946:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827565.300949:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2969 00000800:00000100:0.0:1787827565.300954:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787827565.300954:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827565.300961:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827565.300967:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827565.300977:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011593472/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827565.300999:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827565/real 1787827565] req@ffff907d9114b480 x1874670011593088/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827565.301015:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011593088/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827565.301024:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827565.301027:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827565.301030:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011593088:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827565.301060:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011593216:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827565.301077:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827565.301086:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827565/real 1787827565] req@ffff907d911483c0 x1874670011593216/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827565.301095:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011593216/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827565.301101:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827565.301104:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827565.301106:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011593216:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827565.301110:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011593344:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827565.301122:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827565.301128:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827565/real 1787827565] req@ffff907d9114bc00 x1874670011593344/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827565.301136:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011593344/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827565.301142:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827565.301143:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827565.301145:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011593344:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827565.301149:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011593472:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827565.301160:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827565.301166:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827565/real 1787827565] req@ffff907d91149680 x1874670011593472/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827565.301174:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011593472/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827565.301184:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827565.301187:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827565.301191:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011593472:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827565.428705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827565.428763:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827566.452681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827566.452724:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827567.476752:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827567.476798:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827568.500677:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827568.500729:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827569.524724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827569.524815:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827570.420668:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827570.420676:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827570.420680:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827570.420688:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2974 00000100:00080000:2.0:1787827570.420698:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827570.420704:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827570.420733:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011593600/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827570.420760:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827570.420768:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827570.420770:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827570.420784:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2974 00000100:00080000:2.0:1787827570.420789:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827570.420793:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827570.420803:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827570.420808:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011593600:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827570.420815:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011593728/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827570.420825:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827570.420828:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827570.420829:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827570.420833:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2974 00000100:00080000:2.0:1787827570.420835:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827570.420839:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827570.420843:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827570.420851:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011593856/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827570.420859:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827570.420862:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827570.420864:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787827570.420868:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787827570.420868:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2974 00000100:00080000:2.0:1787827570.420870:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827570.420874:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827570.420877:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827570.420886:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011593984/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827570.420903:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827570/real 1787827570] req@ffff907d9114ad00 x1874670011593600/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827570.420921:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011593600/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827570.420931:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827570.420936:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827570.420940:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011593600:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827570.420970:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011593728:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827570.420989:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827570.420998:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827570/real 1787827570] req@ffff907d9114b840 x1874670011593728/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827570.421006:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011593728/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827570.421012:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827570.421014:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827570.421016:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011593728:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827570.421020:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011593856:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827570.421032:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827570.421037:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827570/real 1787827570] req@ffff907d9114a580 x1874670011593856/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827570.421045:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011593856/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827570.421050:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827570.421052:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827570.421054:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011593856:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827570.421057:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011593984:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827570.421066:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827570.421072:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827570/real 1787827570] req@ffff907d91149a40 x1874670011593984/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827570.421079:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011593984/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827570.421084:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827570.421085:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827570.421087:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011593984:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827570.548714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827570.548808:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827571.572729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827571.572792:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827572.596750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827572.596864:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827573.620733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827573.620851:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827574.644695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827574.644720:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827575.540742:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827575.540752:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827575.540756:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827575.540768:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2979 00000100:00080000:1.0:1787827575.540779:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827575.540787:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827575.540826:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011594112/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827575.540856:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827575.540860:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827575.540862:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827575.540866:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2979 00000100:00080000:1.0:1787827575.540869:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827575.540872:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827575.540883:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827575.540895:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011594240/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827575.540906:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827575.540909:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827575.540911:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827575.540915:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2979 00000100:00080000:1.0:1787827575.540918:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827575.540922:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827575.540927:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827575.540931:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011594112:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827575.540940:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011594368/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827575.540953:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827575.540956:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827575.540958:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827575.540972:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2979 00000100:00080000:1.0:1787827575.540974:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827575.540978:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827575.540982:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827575.540992:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011594496/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827575.541039:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827575.541088:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827575/real 1787827575] req@ffff907d908a21c0 x1874670011594112/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827575.541116:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011594112/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827575.541132:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827575.541138:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827575.541144:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011594112:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827575.541156:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011594240:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827575.541193:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827575.541227:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827575/real 1787827575] req@ffff907d908a2580 x1874670011594240/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827575.541238:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011594240/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827575.541244:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827575.541247:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827575.541249:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011594240:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827575.541268:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011594368:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827575.541285:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827575.541303:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827575/real 1787827575] req@ffff907d908a2940 x1874670011594368/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827575.541316:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011594368/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827575.541322:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827575.541324:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827575.541326:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011594368:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827575.541329:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011594496:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827575.541341:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827575.541368:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827575/real 1787827575] req@ffff907d908a0000 x1874670011594496/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827575.541386:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011594496/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827575.541398:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827575.541403:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827575.541408:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011594496:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827575.668714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827575.668773:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827576.692678:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827576.692764:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827577.716693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827577.716774:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827578.740698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827578.740797:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827579.764725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827579.764819:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827580.660748:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827580.660759:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827580.660764:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827580.660774:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2984 00000100:00080000:0.0:1787827580.660786:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827580.660792:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827580.660845:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011594624/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827580.660873:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827580.660876:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827580.660877:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827580.660881:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2984 00000100:00080000:0.0:1787827580.660884:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827580.660887:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827580.660898:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827580.660911:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011594624:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827580.660915:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011594752/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827580.660930:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827580.660935:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787827580.660937:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827580.660941:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2984 00000100:00080000:0.0:1787827580.660946:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827580.660951:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827580.660955:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827580.660966:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011594880/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827580.660973:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827580.660976:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827580.660978:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827580.660980:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2984 00000100:00080000:0.0:1787827580.660985:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827580.660988:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827580.660992:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827580.661000:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011595008/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827580.661047:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011594752:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827580.661095:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011594880:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827580.661123:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011595008:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827580.788669:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827580.788901:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827581.812726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827581.812792:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827582.836708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827582.836749:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787827583.732831:0:2337:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:2.0:1787827583.732869:0:2337:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787827583.732882:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787827583.732913:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787827583.732927:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787827583.732938:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:1.0:1787827583.732952:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827580/real 1787827583] req@ffff907d90846d00 x1874670011594752/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827583.732979:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011594752/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827583.732993:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827583.732998:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827583.733003:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011594752:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827583.733011:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827580/real 1787827583] req@ffff907d90845680 x1874670011594880/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827583.733020:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011594880/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827583.733026:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827583.733029:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827583.733031:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011594880:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827583.733036:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827580/real 1787827583] req@ffff907d90844780 x1874670011595008/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827583.733043:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011595008/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827583.733049:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827583.733051:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827583.733053:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011595008:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827583.733057:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827580/real 1787827583] req@ffff907d908452c0 x1874670011594624/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827583.733064:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011594624/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827583.733069:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827583.733072:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827583.733075:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011594624:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827583.860741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827583.860817:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827584.884718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827584.884828:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827585.780769:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827585.780781:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827585.780787:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827585.780800:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2989 00000100:00080000:2.0:1787827585.780815:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827585.780825:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827585.780867:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011595136/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827585.780905:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827585.780910:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827585.780913:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827585.780920:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2989 00000100:00080000:2.0:1787827585.780925:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827585.780931:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827585.780945:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827585.780968:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011595264/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827585.780978:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011595136:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827585.780989:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827585.780996:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827585.780998:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827585.781004:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2989 00000100:00080000:2.0:1787827585.781009:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827585.781014:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827585.781020:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827585.781039:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011595392/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827585.781051:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827585.781056:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827585.781058:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827585.781062:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2989 00000100:00080000:2.0:1787827585.781068:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787827585.781075:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787827585.781080:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827585.781089:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827585.781112:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011595520/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827585.781155:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827585/real 1787827585] req@ffff907d9114a940 x1874670011595136/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827585.781181:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011595136/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827585.781196:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827585.781201:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827585.781206:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011595136:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827585.781237:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011595264:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827585.781268:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827585.781284:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827585/real 1787827585] req@ffff907d911492c0 x1874670011595264/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827585.781299:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011595264/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827585.781309:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827585.781313:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827585.781316:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011595264:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827585.781324:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011595392:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827585.781364:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827585.781379:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827585/real 1787827585] req@ffff907d91148f00 x1874670011595392/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827585.781391:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011595392/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827585.781404:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827585.781408:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827585.781412:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011595392:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827585.781420:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011595520:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827585.781444:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827585.781457:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827585/real 1787827585] req@ffff907d91148b40 x1874670011595520/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827585.781476:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011595520/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827585.781485:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827585.781488:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827585.781491:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011595520:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827585.908691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827585.908759:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827586.932695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827586.932765:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827587.956722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827587.956765:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827588.980725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827588.980770:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827590.004741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827590.004863:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827590.900693:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827590.900716:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827590.900728:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827590.900739:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2994 00000100:00080000:1.0:1787827590.900751:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827590.900759:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827590.900808:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011595648/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827590.900839:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827590.900844:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827590.900846:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827590.900851:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2994 00000100:00080000:1.0:1787827590.900855:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827590.900860:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827590.900871:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827590.900885:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011595648:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827590.900890:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011595776/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827590.900925:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827590.900929:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827590.900931:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827590.900936:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2994 00000100:00080000:1.0:1787827590.900940:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827590.900945:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827590.900950:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787827590.900956:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827590.900969:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011595904/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827590.900981:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827590.900985:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827590.900987:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787827590.900992:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827590/real 1787827590] req@ffff907d908a1e00 x1874670011595648/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827590.900993:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2994 00000100:00080000:1.0:1787827590.901001:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827590.901009:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827590.901016:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011595648/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787827590.901016:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1787827590.901030:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827590.901033:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827590.901038:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011596032/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827590.901039:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011595648:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827590.901065:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011595776:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827590.901085:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827590.901094:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827590/real 1787827590] req@ffff907d908a2d00 x1874670011595776/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827590.901102:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011595776/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827590.901109:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827590.901112:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827590.901114:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011595776:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827590.901120:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011595904:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827590.901140:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827590.901163:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827590/real 1787827590] req@ffff907d908a0000 x1874670011595904/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827590.901189:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011595904/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827590.901195:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827590.901198:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827590.901200:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011595904:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827590.901205:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011596032:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827590.901231:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827590.901239:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827590/real 1787827590] req@ffff907d908a2940 x1874670011596032/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827590.901246:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011596032/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827590.901251:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827590.901253:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827590.901255:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011596032:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827591.028675:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827591.028756:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827592.052699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827592.052784:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827593.076701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827593.076866:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827594.100709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827594.100794:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827595.124734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827595.124850:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827596.020809:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827596.020835:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827596.020839:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827596.020848:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 2999 00000100:00080000:3.0:1787827596.020861:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827596.020869:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827596.020910:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011596160/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827596.020944:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827596.020947:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827596.020950:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827596.020953:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2999 00000100:00080000:3.0:1787827596.020957:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827596.020960:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827596.020989:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827596.021008:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011596288/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827596.021023:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011596160:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827596.021025:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827596.021034:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827596.021038:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827596.021062:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2999 00000100:00080000:3.0:1787827596.021066:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827596.021069:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827596.021074:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827596.021093:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011596416/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827596.021108:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827596.021129:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827596.021131:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827596.021136:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 2999 00000100:00080000:3.0:1787827596.021143:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827596.021148:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:0.0:1787827596.021154:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:3.0:1787827596.021157:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827596.021174:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011596544/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827596.021218:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827596/real 1787827596] req@ffff907da562d680 x1874670011596160/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827596.021237:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011596160/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827596.021264:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827596.021269:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827596.021274:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011596160:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827596.021324:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011596288:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827596.021353:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827596.021367:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827596/real 1787827596] req@ffff907da562cf00 x1874670011596288/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827596.021380:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011596288/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827596.021389:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827596.021392:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827596.021396:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011596288:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827596.021403:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011596416:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827596.021419:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827596.021429:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827596/real 1787827596] req@ffff907da562ed00 x1874670011596416/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827596.021447:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011596416/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827596.021471:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827596.021473:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827596.021475:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011596416:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827596.021480:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011596544:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827596.021501:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827596.021514:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827596/real 1787827596] req@ffff907da562c780 x1874670011596544/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827596.021530:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011596544/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827596.021541:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827596.021545:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827596.021549:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011596544:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827596.148694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827596.148758:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827597.172749:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827597.172804:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827598.196718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827598.196753:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827599.220732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827599.220773:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827600.244698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827600.244862:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827601.140799:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827601.140810:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827601.140814:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827601.140823:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3004 00000100:00080000:2.0:1787827601.140834:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827601.140841:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827601.140879:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011596672/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827601.140929:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827601.140932:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827601.140933:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827601.140936:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3004 00000100:00080000:2.0:1787827601.140939:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827601.140942:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827601.140950:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827601.140959:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011596800/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827601.140966:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827601.140968:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827601.140986:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827601.140989:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3004 00000100:00080000:2.0:1787827601.140992:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827601.140996:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827601.140999:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827601.141009:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011596928/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827601.141018:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827601.141022:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827601.141023:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827601.141026:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3004 00000100:00080000:2.0:1787827601.141028:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827601.141031:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827601.141033:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827601.141042:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011597056/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827601.141075:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011596672:192.168.203.104@tcp:250:kworker.0 00000800:00000100:0.0:1787827601.141224:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827601.141332:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827601/real 1787827601] req@ffff907d91148b40 x1874670011596672/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827601.141368:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011596672/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827601.141399:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827601.141418:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827601.141426:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011596672:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827601.141440:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011596800:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827601.141481:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827601.141510:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827601/real 1787827601] req@ffff907d91148f00 x1874670011596800/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827601.141531:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011596800/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827601.141546:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827601.141554:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827601.141560:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011596800:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827601.141571:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011596928:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827601.141679:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827601.141705:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827601/real 1787827601] req@ffff907d911492c0 x1874670011596928/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827601.141739:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011596928/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827601.141754:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827601.141761:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827601.141766:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011596928:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827601.141822:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011597056:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827601.141860:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827601.141881:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827601/real 1787827601] req@ffff907d9114a940 x1874670011597056/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827601.141902:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011597056/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827601.141929:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827601.141935:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827601.141941:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011597056:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827601.268771:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827601.268827:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827602.292678:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827602.292732:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827603.316689:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827603.316744:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827604.340716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827604.340771:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827605.364768:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827605.364787:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827606.260680:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827606.260688:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827606.260692:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827606.260700:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3010 00000100:00080000:2.0:1787827606.260710:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827606.260716:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827606.260750:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011597184/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827606.260780:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827606.260783:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827606.260784:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827606.260787:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3010 00000100:00080000:2.0:1787827606.260790:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827606.260793:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827606.260801:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827606.260813:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011597312/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827606.260821:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827606.260825:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787827606.260826:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011597184:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827606.260828:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827606.260837:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3010 00000100:00080000:2.0:1787827606.260841:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827606.260846:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827606.260851:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827606.260864:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011597440/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827606.260875:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827606.260880:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827606.260882:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827606.260886:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3010 00000100:00080000:2.0:1787827606.260889:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827606.260892:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827606.260896:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787827606.260898:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827606.260908:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011597568/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827606.260936:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827606/real 1787827606] req@ffff907d91149a40 x1874670011597184/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827606.260950:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011597184/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827606.260958:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827606.260961:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827606.260964:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011597184:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827606.260982:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011597312:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827606.261011:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827606.261025:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827606/real 1787827606] req@ffff907d9114a580 x1874670011597312/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827606.261039:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011597312/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827606.261049:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827606.261052:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827606.261054:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011597312:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827606.261059:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011597440:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827606.261080:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827606.261093:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827606/real 1787827606] req@ffff907d9114b840 x1874670011597440/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827606.261106:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011597440/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827606.261115:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827606.261117:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827606.261120:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011597440:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827606.261124:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011597568:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827606.261139:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827606.261146:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827606/real 1787827606] req@ffff907d9114ad00 x1874670011597568/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827606.261153:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011597568/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827606.261163:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827606.261165:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827606.261167:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011597568:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827606.388715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827606.388784:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827607.412687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827607.412722:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827608.436694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827608.436738:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827609.460812:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827609.460916:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827610.484727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827610.484813:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827611.380719:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827611.380731:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827611.380736:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827611.380747:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3015 00000100:00080000:2.0:1787827611.380759:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827611.380767:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827611.380826:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011597696/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827611.380864:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827611.380868:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827611.380871:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827611.380876:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3015 00000100:00080000:2.0:1787827611.380881:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827611.380886:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827611.380898:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827611.380910:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011597696:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827611.380932:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011597824/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827611.380953:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827611.380960:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827611.380963:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827611.380970:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3015 00000100:00080000:2.0:1787827611.380977:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827611.380985:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:0.0:1787827611.380991:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:2.0:1787827611.381001:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827611.381022:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011597952/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827611.381041:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827611/real 1787827611] req@ffff907d91149680 x1874670011597696/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827611.381056:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827611.381063:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827611.381066:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827611.381074:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3015 00000100:00100000:0.0:1787827611.381075:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011597696/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827611.381079:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827611.381090:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00080000:0.0:1787827611.381099:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1787827611.381100:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1787827611.381111:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827611.381123:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011598080/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827611.381124:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011597696:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827611.381161:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011597824:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827611.381237:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827611.381252:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827611/real 1787827611] req@ffff907d9114bc00 x1874670011597824/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827611.381266:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011597824/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827611.381285:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827611.381289:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827611.381292:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011597824:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827611.381300:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011597952:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827611.381319:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827611.381329:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827611/real 1787827611] req@ffff907d911483c0 x1874670011597952/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827611.381340:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011597952/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827611.381359:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827611.381362:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827611.381365:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011597952:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827611.381371:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011598080:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827611.381386:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827611.381406:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827611/real 1787827611] req@ffff907d9114b480 x1874670011598080/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827611.381429:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011598080/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827611.381437:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827611.381440:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827611.381443:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011598080:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827611.508785:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827611.508887:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827612.532709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827612.532939:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827613.556726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827613.556815:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827614.580792:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827614.580900:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827615.604767:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827615.604870:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827616.500788:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827616.500801:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827616.500807:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827616.500820:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3020 00000100:00080000:0.0:1787827616.500835:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827616.500845:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827616.500894:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011598208/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827616.500940:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827616.500946:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827616.500949:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827616.500954:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3020 00000100:00080000:0.0:1787827616.500958:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827616.500962:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827616.500974:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827616.500994:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011598336/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827616.501015:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827616.501022:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787827616.501026:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827616.501035:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3020 00000100:00080000:0.0:1787827616.501044:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827616.501053:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827616.501063:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827616.501075:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011598208:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827616.501094:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011598464/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827616.501131:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827616.501164:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827616.501168:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827616.501178:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3020 00000100:00080000:0.0:1787827616.501200:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787827616.501214:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787827616.501215:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827616.501234:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827616.501266:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011598592/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827616.501280:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827616/real 1787827616] req@ffff907d90844000 x1874670011598208/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827616.501315:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011598208/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827616.501331:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827616.501337:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827616.501343:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011598208:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827616.501352:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011598336:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827616.501379:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827616.501394:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827616/real 1787827616] req@ffff907d908443c0 x1874670011598336/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827616.501417:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011598336/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827616.501428:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827616.501431:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827616.501435:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011598336:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827616.501463:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011598464:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827616.501486:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827616.501506:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827616/real 1787827616] req@ffff907d90844b40 x1874670011598464/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827616.501530:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011598464/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827616.501547:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827616.501553:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827616.501560:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011598464:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827616.501573:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011598592:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827616.501654:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827616.501674:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827616/real 1787827616] req@ffff907d908470c0 x1874670011598592/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827616.501689:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011598592/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827616.501708:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827616.501712:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827616.501716:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011598592:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827616.628740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827616.628896:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827617.652740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827617.652883:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827618.676751:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827618.676869:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827619.700672:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827619.700733:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827620.724732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827620.724807:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827621.620692:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827621.620699:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827621.620703:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827621.620713:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3025 00000100:00080000:2.0:1787827621.620723:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827621.620729:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827621.620760:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011598720/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827621.620796:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827621.620800:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827621.620801:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827621.620804:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3025 00000100:00080000:2.0:1787827621.620807:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827621.620810:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827621.620818:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827621.620830:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011598848/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827621.620840:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787827621.620842:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011598720:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827621.620845:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827621.620850:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827621.620854:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3025 00000100:00080000:2.0:1787827621.620857:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827621.620861:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827621.620865:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827621.620874:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011598976/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827621.620892:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827621.620895:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827621.620896:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827621.620899:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3025 00000100:00080000:2.0:1787827621.620902:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827621.620906:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827621.620910:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787827621.620917:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827621.620921:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011599104/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827621.620973:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827621/real 1787827621] req@ffff907d9114b480 x1874670011598720/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827621.620989:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011598720/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827621.620999:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827621.621005:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827621.621010:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011598720:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827621.621035:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011598848:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827621.621061:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827621.621072:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827621/real 1787827621] req@ffff907d911483c0 x1874670011598848/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827621.621082:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011598848/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827621.621088:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827621.621090:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827621.621093:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011598848:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827621.621097:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011598976:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827621.621132:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827621.621140:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827621/real 1787827621] req@ffff907d9114bc00 x1874670011598976/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827621.621159:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011598976/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827621.621168:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827621.621172:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827621.621176:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011598976:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827621.621183:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011599104:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827621.621208:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827621.621223:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827621/real 1787827621] req@ffff907d91149680 x1874670011599104/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827621.621236:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011599104/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827621.621252:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827621.621256:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827621.621260:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011599104:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827621.748703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827621.748794:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827622.772803:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827622.772899:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827623.796715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827623.796866:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827624.820689:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827624.820736:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827625.844829:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827625.845008:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827626.740726:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827626.740737:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827626.740741:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827626.740751:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3030 00000100:00080000:2.0:1787827626.740762:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827626.740770:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827626.740809:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011599232/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827626.740838:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827626.740842:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827626.740844:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827626.740847:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3030 00000100:00080000:2.0:1787827626.740851:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827626.740855:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827626.740866:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827626.740897:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011599360/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827626.740918:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011599232:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827626.740940:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827626.740945:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827626.740947:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827626.740955:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3030 00000100:00080000:2.0:1787827626.740962:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827626.740967:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827626.740974:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827626.741014:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011599488/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827626.741023:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827626.741026:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827626.741028:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827626.741031:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3030 00000100:00080000:2.0:1787827626.741035:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827626.741039:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827626.741046:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787827626.741054:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827626.741063:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011599616/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827626.741130:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827626/real 1787827626] req@ffff907d9114ad00 x1874670011599232/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827626.741159:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011599232/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827626.741176:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827626.741183:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827626.741188:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011599232:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827626.741226:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011599360:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827626.741295:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827626.741322:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827626/real 1787827626] req@ffff907d9114b840 x1874670011599360/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827626.741346:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011599360/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827626.741361:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827626.741368:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827626.741374:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011599360:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827626.741388:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011599488:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827626.741428:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827626.741461:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827626/real 1787827626] req@ffff907d9114a580 x1874670011599488/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827626.741476:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011599488/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827626.741486:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827626.741489:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827626.741493:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011599488:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827626.741512:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011599616:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827626.741563:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827626.741572:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827626/real 1787827626] req@ffff907d91149a40 x1874670011599616/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827626.741582:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011599616/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827626.741590:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827626.741593:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827626.741595:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011599616:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827626.868721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827626.868841:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827627.892752:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827627.892784:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827628.916719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827628.916765:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827629.940742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827629.940801:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827630.964739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827630.964914:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827631.860769:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827631.860783:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827631.860788:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827631.860801:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3035 00000100:00080000:1.0:1787827631.860818:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827631.860830:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827631.860888:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011599744/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827631.860943:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827631.860949:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827631.860952:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827631.860957:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3035 00000100:00080000:1.0:1787827631.860963:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827631.860969:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827631.860982:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827631.861003:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011599872/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827631.861021:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787827631.861025:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011599744:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827631.861026:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827631.861084:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827631.861092:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3035 00000100:00080000:1.0:1787827631.861097:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827631.861102:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827631.861109:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827631.861152:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011600000/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827631.861172:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827631.861179:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827631.861183:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827631.861190:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3035 00000100:00080000:1.0:1787827631.861199:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827631.861208:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827631.861236:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787827631.861240:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827631.861264:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011600128/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827631.861303:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827631/real 1787827631] req@ffff907d908a2580 x1874670011599744/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827631.861333:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011599744/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827631.861352:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827631.861361:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827631.861368:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011599744:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827631.861405:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011599872:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827631.861445:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827631.861463:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827631/real 1787827631] req@ffff907d908a21c0 x1874670011599872/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827631.861479:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011599872/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827631.861489:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827631.861493:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827631.861497:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011599872:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827631.861505:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011600000:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827631.861527:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827631.861539:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827631/real 1787827631] req@ffff907d908a0f00 x1874670011600000/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827631.861553:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011600000/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827631.861579:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827631.861585:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827631.861592:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011600000:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827631.861603:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011600128:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827631.861700:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827631.861724:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827631/real 1787827631] req@ffff907d908a12c0 x1874670011600128/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827631.861748:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011600128/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827631.861763:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827631.861768:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827631.861772:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011600128:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827631.988764:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827631.988866:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827633.012769:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827633.012826:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827634.036733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827634.036821:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827635.060681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827635.060721:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827636.084676:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827636.084723:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827636.980712:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827636.980723:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827636.980728:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827636.980740:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3040 00000100:00080000:1.0:1787827636.980752:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827636.980759:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827636.980811:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011600256/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827636.980861:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827636.980866:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827636.980869:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827636.980875:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3040 00000100:00080000:1.0:1787827636.980881:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827636.980888:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827636.980903:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827636.980918:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011600384/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827636.980928:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011600256:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827636.980960:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827636.980963:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827636.980966:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827636.980972:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3040 00000100:00080000:1.0:1787827636.980976:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827636.980979:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827636.980986:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827636.981015:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011600512/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827636.981024:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827636.981028:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1787827636.981046:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787827636.981048:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827636.981054:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3040 00000100:00080000:1.0:1787827636.981059:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827636.981063:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827636.981070:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827636.981081:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011600640/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827636.981090:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827636/real 1787827636] req@ffff907d908a1680 x1874670011600256/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827636.981107:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011600256/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827636.981116:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827636.981123:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827636.981126:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011600256:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827636.981148:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011600384:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827636.981165:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827636.981174:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827636/real 1787827636] req@ffff907d908a3840 x1874670011600384/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827636.981182:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011600384/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827636.981188:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827636.981190:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827636.981192:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011600384:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827636.981197:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011600512:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827636.981207:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827636.981213:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827636/real 1787827636] req@ffff907d908a12c0 x1874670011600512/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827636.981220:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011600512/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827636.981225:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827636.981227:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827636.981229:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011600512:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827636.981233:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011600640:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827636.981256:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827636.981261:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827636/real 1787827636] req@ffff907d908a0f00 x1874670011600640/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827636.981269:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011600640/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827636.981274:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827636.981276:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827636.981278:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011600640:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827637.108727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827637.108859:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827638.132838:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827638.132867:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827639.156683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827639.156738:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827640.180685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827640.180754:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827641.204690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827641.204840:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827642.100819:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827642.100827:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827642.100831:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827642.100840:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3045 00000100:00080000:2.0:1787827642.100849:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827642.100855:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827642.100889:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011600768/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827642.100913:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827642.100916:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827642.100917:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827642.100920:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3045 00000100:00080000:2.0:1787827642.100922:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827642.100924:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827642.100932:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827642.100941:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011600896/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827642.100949:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827642.100952:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827642.100953:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827642.100956:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3045 00000100:00080000:2.0:1787827642.100974:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827642.100981:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827642.100983:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011600768:192.168.203.104@tcp:250:kworker.0 00000008:00100000:2.0:1787827642.100985:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827642.100998:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011601024/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827642.101006:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827642.101008:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827642.101010:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827642.101012:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3045 00000100:00080000:2.0:1787827642.101014:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827642.101017:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827642.101020:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827642.101028:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011601152/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827642.101054:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827642.101096:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827642/real 1787827642] req@ffff907d9114a940 x1874670011600768/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827642.101124:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011600768/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827642.101136:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827642.101140:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827642.101145:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011600768:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827642.101152:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011600896:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827642.101177:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827642.101189:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827642/real 1787827642] req@ffff907d911492c0 x1874670011600896/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827642.101202:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011600896/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827642.101217:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827642.101221:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827642.101224:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011600896:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827642.101249:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011601024:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827642.101271:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827642.101282:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827642/real 1787827642] req@ffff907d91148f00 x1874670011601024/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827642.101296:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011601024/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827642.101305:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827642.101309:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827642.101313:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011601024:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827642.101319:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011601152:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827642.101338:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827642.101349:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827642/real 1787827642] req@ffff907d91148b40 x1874670011601152/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827642.101361:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011601152/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827642.101370:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827642.101373:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827642.101378:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011601152:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827642.228707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827642.228771:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827643.252773:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827643.252920:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827644.276765:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827644.276831:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827645.300705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827645.300764:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827646.324696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827646.324802:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827647.220687:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827647.220696:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827647.220701:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827647.220710:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3050 00000100:00080000:2.0:1787827647.220724:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827647.220732:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827647.220776:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011601280/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827647.220829:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827647.220836:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827647.220839:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827647.220848:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3050 00000100:00080000:2.0:1787827647.220856:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827647.220864:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827647.220880:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827647.220903:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011601408/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827647.220904:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011601280:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827647.220949:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827647.220956:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827647.220959:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827647.220965:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3050 00000100:00080000:2.0:1787827647.220987:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827647.220991:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827647.220997:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827647.221009:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011601536/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827647.221021:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827647.221025:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827647.221028:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827647.221033:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3050 00000100:00080000:2.0:1787827647.221036:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827647.221041:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827647.221046:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827647.221058:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011601664/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827647.221070:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011601408:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827647.221126:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011601536:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827647.221154:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011601664:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827647.348697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827647.348770:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827648.372700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827648.372771:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827649.396715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827649.396772:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787827650.293025:0:2338:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:2.0:1787827650.293070:0:2338:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787827650.293088:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787827650.293133:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787827650.293144:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787827650.293155:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:0.0:1787827650.293191:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827647/real 1787827650] req@ffff907d9114a940 x1874670011601664/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827650.293220:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011601664/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827650.293234:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827650.293240:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827650.293244:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011601664:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827650.293252:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827647/real 1787827650] req@ffff907d91148f00 x1874670011601408/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827650.293264:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011601408/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827650.293273:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827650.293276:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827650.293279:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011601408:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827650.293286:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827647/real 1787827650] req@ffff907d911492c0 x1874670011601536/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827650.293298:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011601536/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827650.293307:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827650.293310:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827650.293313:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011601536:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827650.293320:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827647/real 1787827650] req@ffff907d91148b40 x1874670011601280/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827650.293331:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011601280/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827650.293338:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827650.293340:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827650.293343:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011601280:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827650.420692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827650.420756:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827651.444716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827651.444742:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827652.340694:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827652.340702:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827652.340706:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827652.340714:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3056 00000100:00080000:2.0:1787827652.340725:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827652.340731:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827652.340761:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011601792/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827652.340786:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827652.340788:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827652.340790:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827652.340793:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3056 00000100:00080000:2.0:1787827652.340795:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827652.340797:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827652.340804:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827652.340812:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011601920/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827652.340819:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827652.340821:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827652.340823:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827652.340825:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3056 00000100:00080000:2.0:1787827652.340826:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827652.340829:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827652.340831:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827652.340837:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011602048/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827652.340843:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827652.340845:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827652.340846:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827652.340848:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3056 00000100:00080000:2.0:1787827652.340851:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827652.340854:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827652.340857:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827652.340864:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011602176/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827652.340883:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011601792:192.168.203.104@tcp:250:kworker.0 00000800:00000100:0.0:1787827652.340963:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827652.341008:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827652/real 1787827652] req@ffff907d91149a40 x1874670011601792/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827652.341041:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011601792/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827652.341053:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827652.341057:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827652.341061:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011601792:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827652.341070:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011601920:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827652.341094:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827652.341108:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827652/real 1787827652] req@ffff907d9114a580 x1874670011601920/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827652.341121:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011601920/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827652.341131:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827652.341134:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827652.341136:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011601920:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827652.341143:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011602048:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827652.341163:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827652.341174:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827652/real 1787827652] req@ffff907d9114b840 x1874670011602048/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827652.341186:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011602048/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827652.341195:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827652.341199:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827652.341202:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011602048:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827652.341237:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011602176:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827652.341259:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827652.341270:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827652/real 1787827652] req@ffff907d9114ad00 x1874670011602176/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827652.341282:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011602176/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827652.341289:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827652.341292:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827652.341295:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011602176:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827652.468693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827652.468747:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827653.492720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827653.492804:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827654.516682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827654.516770:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827655.540693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827655.540761:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827656.564665:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827656.564784:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827657.460718:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827657.460727:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827657.460731:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827657.460739:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3061 00000100:00080000:2.0:1787827657.460751:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827657.460759:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827657.460790:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011602304/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827657.460819:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827657.460822:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827657.460823:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827657.460826:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3061 00000100:00080000:2.0:1787827657.460830:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827657.460832:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827657.460842:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827657.460871:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011602432/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827657.460885:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827657.460889:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787827657.460891:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011602304:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827657.460892:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827657.460907:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3061 00000100:00080000:2.0:1787827657.460911:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827657.460917:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827657.460925:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827657.460939:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011602560/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827657.460950:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827657.460953:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827657.460957:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827657.460960:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3061 00000100:00080000:2.0:1787827657.460965:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827657.460970:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827657.460974:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827657.460983:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011602688/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827657.460998:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827657.461051:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827657/real 1787827657] req@ffff907d91149680 x1874670011602304/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827657.461073:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011602304/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827657.461083:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827657.461087:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827657.461092:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011602304:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827657.461115:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011602432:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827657.461133:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827657.461142:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827657/real 1787827657] req@ffff907d9114bc00 x1874670011602432/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827657.461152:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011602432/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827657.461158:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827657.461161:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827657.461164:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011602432:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827657.461168:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011602560:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827657.461182:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827657.461190:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827657/real 1787827657] req@ffff907d911483c0 x1874670011602560/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827657.461199:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011602560/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827657.461211:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827657.461214:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827657.461217:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011602560:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827657.461223:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011602688:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827657.461245:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827657.461256:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827657/real 1787827657] req@ffff907d9114b480 x1874670011602688/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827657.461266:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011602688/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827657.461277:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827657.461280:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827657.461283:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011602688:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827657.588727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827657.588801:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827658.612677:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827658.612774:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827659.636737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827659.636789:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827660.660679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827660.660736:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827661.684708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827661.684759:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827662.580705:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827662.580713:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827662.580717:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827662.580726:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3066 00000100:00080000:2.0:1787827662.580736:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827662.580742:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827662.580775:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011602816/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827662.580793:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827662.580796:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827662.580797:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827662.580800:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3066 00000100:00080000:2.0:1787827662.580802:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827662.580804:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827662.580812:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827662.580839:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011602944/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827662.580843:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011602816:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827662.580857:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827662.580859:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827662.580860:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827662.580864:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3066 00000100:00080000:2.0:1787827662.580867:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827662.580870:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827662.580874:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827662.580883:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011603072/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827662.580890:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827662.580893:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827662.580894:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827662.580897:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3066 00000100:00080000:2.0:1787827662.580899:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827662.580901:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:1.0:1787827662.580905:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:2.0:1787827662.580905:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827662.580917:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011603200/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827662.580942:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827662/real 1787827662] req@ffff907d9114b480 x1874670011602816/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827662.580957:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011602816/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827662.580964:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827662.580967:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827662.580970:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011602816:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827662.580988:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011602944:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827662.581003:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827662.581011:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827662/real 1787827662] req@ffff907d911483c0 x1874670011602944/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827662.581020:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011602944/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827662.581026:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827662.581028:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827662.581030:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011602944:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827662.581034:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011603072:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827662.581044:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827662.581050:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827662/real 1787827662] req@ffff907d9114bc00 x1874670011603072/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827662.581057:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011603072/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827662.581063:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827662.581064:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827662.581066:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011603072:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827662.581070:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011603200:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827662.581080:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827662.581085:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827662/real 1787827662] req@ffff907d91149680 x1874670011603200/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827662.581093:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011603200/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827662.581098:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827662.581100:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827662.581102:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011603200:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827662.708709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827662.708770:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827663.732754:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827663.732818:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827664.756775:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827664.756858:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787827665.780687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827665.780737:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827666.804704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827666.804755:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827667.700723:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827667.700733:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827667.700738:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827667.700748:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3071 00000100:00080000:1.0:1787827667.700760:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827667.700768:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827667.700816:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011603328/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827667.700862:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827667.700867:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827667.700870:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827667.700876:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3071 00000100:00080000:1.0:1787827667.700880:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827667.700885:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827667.700890:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011603328:192.168.203.104@tcp:250:kworker.0 00000008:00100000:1.0:1787827667.700897:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827667.700912:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011603456/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827667.700932:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827667.700936:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827667.700938:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827667.700941:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3071 00000100:00080000:1.0:1787827667.700946:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827667.700950:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827667.700955:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787827667.700956:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827667.700970:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011603584/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827667.701000:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827667/real 1787827667] req@ffff907d908a21c0 x1874670011603328/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827667.701002:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827667.701012:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827667.701016:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827667.701022:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3071 00000100:00100000:0.0:1787827667.701027:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011603328/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827667.701027:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827667.701036:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00080000:0.0:1787827667.701040:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1787827667.701044:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1787827667.701046:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827667.701053:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011603328:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827667.701062:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011603712/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827667.701079:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011603456:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827667.701105:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827667.701115:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827667/real 1787827667] req@ffff907d908a2580 x1874670011603456/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827667.701123:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011603456/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827667.701133:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827667.701136:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827667.701138:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011603456:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827667.701142:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011603584:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827667.701154:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827667.701160:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827667/real 1787827667] req@ffff907d908a2940 x1874670011603584/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827667.701167:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011603584/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827667.701173:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827667.701174:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827667.701185:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011603584:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827667.701196:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011603712:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827667.701208:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827667.701214:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827667/real 1787827667] req@ffff907d908a0000 x1874670011603712/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827667.701221:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011603712/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827667.701227:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827667.701228:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827667.701231:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011603712:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827667.828692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827667.828766:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827668.852699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827668.852752:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827669.876680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827669.876805:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827670.900705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827670.900755:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827671.924696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827671.924744:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827672.820736:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827672.820756:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827672.820762:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827672.820770:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3076 00000100:00080000:0.0:1787827672.820780:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827672.820786:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827672.820820:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011603840/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827672.820848:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827672.820852:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827672.820854:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827672.820859:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3076 00000100:00080000:0.0:1787827672.820864:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827672.820870:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827672.820879:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827672.820886:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011603840:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827672.820893:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011603968/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827672.820902:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827672.820905:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787827672.820906:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827672.820909:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3076 00000100:00080000:0.0:1787827672.820917:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827672.820921:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827672.820925:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827672.820933:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011604096/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827672.820941:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827672.820943:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827672.820945:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787827672.820948:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787827672.820951:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3076 00000100:00080000:0.0:1787827672.820956:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827672.820959:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827672.820968:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827672.820978:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011604224/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827672.820995:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827672/real 1787827672] req@ffff907d90845e00 x1874670011603840/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827672.821024:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011603840/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827672.821034:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827672.821037:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827672.821040:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011603840:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827672.821059:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011603968:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827672.821077:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827672.821086:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827672/real 1787827672] req@ffff907d90846580 x1874670011603968/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827672.821094:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011603968/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827672.821100:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827672.821102:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827672.821104:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011603968:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827672.821107:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011604096:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827672.821118:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827672.821124:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827672/real 1787827672] req@ffff907d90847c00 x1874670011604096/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827672.821131:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011604096/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827672.821141:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827672.821143:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827672.821145:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011604096:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827672.821148:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011604224:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827672.821157:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827672.821162:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827672/real 1787827672] req@ffff907d908470c0 x1874670011604224/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827672.821169:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011604224/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827672.821174:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827672.821176:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827672.821178:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011604224:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827672.948718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827672.948818:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827673.972681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827673.972837:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827674.996723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827674.996727:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827676.020668:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827676.020742:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827677.044681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827677.044769:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827677.940789:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827677.940798:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827677.940802:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827677.940812:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3081 00000100:00080000:2.0:1787827677.940822:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827677.940830:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827677.940897:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011604352/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827677.940928:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827677.940932:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827677.940934:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827677.940938:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3081 00000100:00080000:2.0:1787827677.940941:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827677.940962:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827677.940972:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827677.940987:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011604480/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827677.941001:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011604352:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827677.941020:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827677.941024:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827677.941026:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827677.941031:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3081 00000100:00080000:2.0:1787827677.941034:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827677.941039:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827677.941045:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827677.941058:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011604608/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827677.941068:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787827677.941074:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827677.941079:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827677.941081:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827677.941087:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3081 00000100:00080000:2.0:1787827677.941091:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827677.941096:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827677.941102:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827677.941108:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827677/real 1787827677] req@ffff907d9114ad00 x1874670011604352/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827677.941119:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011604736/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827677.941175:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011604352/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827677.941187:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827677.941190:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827677.941193:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011604352:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827677.941217:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011604480:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827677.941242:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827677.941256:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827677/real 1787827677] req@ffff907d9114b840 x1874670011604480/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827677.941269:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011604480/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827677.941277:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827677.941280:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827677.941284:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011604480:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827677.941290:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011604608:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827677.941309:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827677.941319:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827677/real 1787827677] req@ffff907d9114a580 x1874670011604608/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827677.941331:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011604608/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827677.941339:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827677.941342:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827677.941345:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011604608:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827677.941351:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011604736:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827677.941366:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827677.941377:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827677/real 1787827677] req@ffff907d91149a40 x1874670011604736/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827677.941388:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011604736/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827677.941395:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827677.941397:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827677.941402:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011604736:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827678.068701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827678.068851:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827679.092698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827679.092855:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827680.116674:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827680.116881:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827681.140697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827681.140840:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827682.164724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827682.164793:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827683.060681:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827683.060692:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827683.060696:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827683.060704:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3086 00000100:00080000:2.0:1787827683.060714:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827683.060720:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827683.060763:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011604864/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827683.060785:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827683.060788:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827683.060789:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827683.060792:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3086 00000100:00080000:2.0:1787827683.060795:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827683.060798:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827683.060806:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827683.060815:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011604992/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827683.060824:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787827683.060825:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011604864:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827683.060829:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827683.060833:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827683.060837:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3086 00000100:00080000:2.0:1787827683.060839:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827683.060841:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827683.060845:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827683.060853:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011605120/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827683.060859:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827683.060862:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827683.060864:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827683.060866:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3086 00000100:00080000:2.0:1787827683.060868:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827683.060870:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827683.060873:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827683.060881:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011605248/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827683.060899:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827683.060934:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827683/real 1787827683] req@ffff907d91148b40 x1874670011604864/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827683.060947:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011604864/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827683.060955:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827683.060958:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827683.060960:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011604864:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827683.060977:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011604992:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827683.060993:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827683.061001:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827683/real 1787827683] req@ffff907d911492c0 x1874670011604992/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827683.061009:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011604992/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827683.061020:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827683.061023:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827683.061027:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011604992:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827683.061032:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011605120:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827683.061052:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827683.061063:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827683/real 1787827683] req@ffff907d91148f00 x1874670011605120/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827683.061077:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011605120/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827683.061087:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827683.061090:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827683.061093:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011605120:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827683.061100:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011605248:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827683.061121:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827683.061132:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827683/real 1787827683] req@ffff907d9114a940 x1874670011605248/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827683.061140:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011605248/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827683.061145:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827683.061148:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827683.061151:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011605248:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827683.188712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827683.188788:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827684.212750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827684.212838:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827685.236709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827685.236773:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827686.260747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827686.260829:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827687.284791:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827687.284976:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827688.180710:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827688.180718:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827688.180722:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827688.180748:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3091 00000100:00080000:2.0:1787827688.180759:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827688.180766:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827688.180808:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011605376/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827688.180837:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827688.180840:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827688.180842:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827688.180846:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3091 00000100:00080000:2.0:1787827688.180849:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827688.180852:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827688.180864:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827688.180883:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011605504/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827688.180914:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787827688.180917:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011605376:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827688.180919:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827688.180924:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827688.180932:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3091 00000100:00080000:2.0:1787827688.180937:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827688.180941:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827688.180946:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827688.180957:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011605632/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827688.180965:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827688.180968:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827688.180970:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827688.180972:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3091 00000100:00080000:2.0:1787827688.180975:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827688.180978:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827688.180982:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827688.180991:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011605760/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827688.181000:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827688.181063:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827688/real 1787827688] req@ffff907d9114a940 x1874670011605376/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827688.181082:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011605376/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827688.181103:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827688.181107:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827688.181111:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011605376:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827688.181134:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011605504:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827688.181159:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827688.181172:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827688/real 1787827688] req@ffff907d91148f00 x1874670011605504/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827688.181183:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011605504/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827688.181191:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827688.181194:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827688.181198:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011605504:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827688.181203:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011605632:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827688.181218:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827688.181227:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827688/real 1787827688] req@ffff907d911492c0 x1874670011605632/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827688.181238:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011605632/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827688.181246:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827688.181248:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827688.181251:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011605632:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827688.181256:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011605760:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827688.181268:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827688.181276:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827688/real 1787827688] req@ffff907d91148b40 x1874670011605760/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827688.181286:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011605760/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827688.181293:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827688.181296:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827688.181298:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011605760:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827688.308744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827688.308797:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827689.332696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827689.332749:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827690.356733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827690.356790:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827691.380727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827691.380787:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827692.404711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827692.404891:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827693.300758:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827693.300769:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827693.300775:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827693.300787:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3097 00000100:00080000:2.0:1787827693.300799:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827693.300809:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827693.300850:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011605888/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827693.300926:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827693.300931:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827693.300934:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827693.300942:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3097 00000100:00080000:2.0:1787827693.300957:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827693.300964:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827693.300981:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827693.301009:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011605888:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827693.301009:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011606016/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827693.301039:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827693.301045:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827693.301060:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827693.301068:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3097 00000100:00080000:2.0:1787827693.301092:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827693.301097:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827693.301105:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827693.301125:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011606144/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827693.301159:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787827693.301170:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827693.301185:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827693.301189:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827693.301199:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3097 00000100:00080000:2.0:1787827693.301211:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827693.301217:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827693.301226:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827693.301260:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827693/real 1787827693] req@ffff907d91149a40 x1874670011605888/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827693.301260:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011606272/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827693.301335:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011605888/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827693.301348:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827693.301354:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827693.301361:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011605888:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827693.301415:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011606016:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827693.301450:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827693.301466:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827693/real 1787827693] req@ffff907d9114a580 x1874670011606016/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827693.301481:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011606016/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827693.301491:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827693.301513:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827693.301517:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011606016:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827693.301533:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011606144:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827693.301564:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827693.301579:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827693/real 1787827693] req@ffff907d9114b840 x1874670011606144/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827693.301596:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011606144/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827693.301708:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827693.301716:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827693.301722:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011606144:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827693.301746:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011606272:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827693.301782:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827693.301812:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827693/real 1787827693] req@ffff907d9114ad00 x1874670011606272/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827693.301825:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011606272/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827693.301834:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827693.301837:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827693.301840:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011606272:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827693.428696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827693.428872:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827694.452734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827694.452791:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827695.476693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827695.476765:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827696.500710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827696.500765:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827697.524727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827697.524766:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827698.420699:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827698.420706:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827698.420710:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827698.420719:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3102 00000100:00080000:2.0:1787827698.420728:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827698.420734:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827698.420764:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011606400/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827698.420785:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827698.420788:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827698.420790:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827698.420793:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3102 00000100:00080000:2.0:1787827698.420795:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827698.420799:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827698.420807:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827698.420817:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011606400:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827698.420818:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011606528/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827698.420831:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827698.420833:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827698.420834:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827698.420836:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3102 00000100:00080000:2.0:1787827698.420838:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827698.420841:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827698.420844:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827698.420851:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011606656/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827698.420857:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827698.420860:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827698.420861:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827698.420864:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3102 00000100:00080000:2.0:1787827698.420866:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827698.420868:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827698.420870:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827698.420878:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011606784/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827698.420889:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827698.420925:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827698/real 1787827698] req@ffff907d91149680 x1874670011606400/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827698.420937:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011606400/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827698.420944:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827698.420947:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827698.420950:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011606400:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827698.420968:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011606528:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827698.420992:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827698.421005:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827698/real 1787827698] req@ffff907d9114bc00 x1874670011606528/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827698.421018:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011606528/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827698.421032:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827698.421034:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827698.421036:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011606528:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827698.421041:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011606656:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827698.421057:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827698.421064:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827698/real 1787827698] req@ffff907d911483c0 x1874670011606656/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827698.421072:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011606656/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827698.421077:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827698.421079:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827698.421081:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011606656:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827698.421084:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011606784:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827698.421095:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827698.421101:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827698/real 1787827698] req@ffff907d9114b480 x1874670011606784/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827698.421108:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011606784/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827698.421113:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827698.421115:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827698.421117:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011606784:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827698.548698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827698.548750:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827699.572682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827699.572716:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827700.596683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827700.596786:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827701.620739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827701.620829:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827702.644721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827702.644769:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827703.540686:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827703.540694:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827703.540698:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827703.540707:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3107 00000100:00080000:2.0:1787827703.540717:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827703.540723:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827703.540764:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011606912/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827703.540785:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827703.540788:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827703.540790:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827703.540793:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3107 00000100:00080000:2.0:1787827703.540795:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827703.540798:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827703.540806:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827703.540816:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011607040/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827703.540824:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827703.540827:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827703.540829:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787827703.540833:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011606912:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827703.540834:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3107 00000100:00080000:2.0:1787827703.540839:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827703.540843:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827703.540846:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827703.540853:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011607168/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827703.540860:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827703.540862:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827703.540863:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827703.540866:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3107 00000100:00080000:2.0:1787827703.540868:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827703.540870:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827703.540873:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827703.540879:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011607296/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827703.540897:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827703.540937:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827703/real 1787827703] req@ffff907d9114b480 x1874670011606912/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827703.540952:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011606912/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827703.540961:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827703.540964:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827703.540966:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011606912:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827703.540985:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011607040:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827703.541001:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827703.541009:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827703/real 1787827703] req@ffff907d911483c0 x1874670011607040/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827703.541017:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011607040/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827703.541023:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827703.541025:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827703.541028:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011607040:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827703.541031:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011607168:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827703.541042:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827703.541049:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827703/real 1787827703] req@ffff907d9114bc00 x1874670011607168/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827703.541056:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011607168/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827703.541062:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827703.541063:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827703.541065:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011607168:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827703.541069:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011607296:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827703.541079:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827703.541084:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827703/real 1787827703] req@ffff907d91149680 x1874670011607296/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827703.541091:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011607296/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827703.541097:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827703.541098:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827703.541100:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011607296:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827703.668724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827703.668803:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827704.692718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827704.692813:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827705.716712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827705.716807:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827706.740702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827706.740797:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827707.764742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827707.764842:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827708.660772:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827708.660798:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827708.660803:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827708.660812:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3112 00000100:00080000:0.0:1787827708.660822:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827708.660828:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827708.660861:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011607424/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827708.660885:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827708.660888:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827708.660889:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827708.660892:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3112 00000100:00080000:0.0:1787827708.660894:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827708.660896:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827708.660905:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827708.660913:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011607552/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827708.660922:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827708.660926:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787827708.660926:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011607424:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787827708.660930:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827708.660936:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3112 00000100:00080000:0.0:1787827708.660938:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827708.660941:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827708.660944:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827708.660951:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011607680/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827708.660958:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827708.660960:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827708.660962:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827708.660965:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3112 00000100:00080000:0.0:1787827708.660967:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827708.660970:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827708.660973:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827708.660980:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011607808/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787827708.660996:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827708.661035:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827708/real 1787827708] req@ffff907d90844b40 x1874670011607424/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827708.661051:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011607424/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827708.661060:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827708.661063:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827708.661066:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011607424:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827708.661083:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011607552:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787827708.661100:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827708.661108:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827708/real 1787827708] req@ffff907d908443c0 x1874670011607552/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827708.661116:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011607552/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827708.661122:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827708.661124:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827708.661126:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011607552:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827708.661129:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011607680:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827708.661140:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827708.661146:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827708/real 1787827708] req@ffff907d90844000 x1874670011607680/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827708.661154:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011607680/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827708.661170:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827708.661171:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827708.661173:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011607680:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827708.661177:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011607808:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787827708.661187:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827708.661193:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827708/real 1787827708] req@ffff907d908452c0 x1874670011607808/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827708.661200:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011607808/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787827708.661205:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787827708.661207:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787827708.661209:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011607808:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827708.788696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827708.788736:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827709.812693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827709.812745:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827710.836726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827710.836832:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827711.860711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827711.860799:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827712.884718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827712.884853:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827713.780711:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827713.780721:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827713.780726:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827713.780735:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3117 00000100:00080000:2.0:1787827713.780746:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827713.780753:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827713.780786:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011607936/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827713.780811:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827713.780815:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827713.780816:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827713.780820:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3117 00000100:00080000:2.0:1787827713.780823:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827713.780826:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827713.780835:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827713.780846:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011608064/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827713.780856:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827713.780862:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787827713.780864:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011607936:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827713.780865:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827713.780871:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3117 00000100:00080000:2.0:1787827713.780875:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827713.780882:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827713.780887:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827713.780896:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011608192/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827713.780903:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827713.780906:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827713.780908:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827713.780911:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3117 00000100:00080000:2.0:1787827713.780914:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827713.780916:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827713.780920:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827713.780929:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011608320/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787827713.781007:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011608064:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787827713.781044:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011608192:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827713.781065:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011608320:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827713.908779:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827713.908899:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827714.932690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827714.932775:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827715.956707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827715.956770:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787827716.852966:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787827716.853026:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787827716.853046:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787827716.853113:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787827716.853125:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787827716.853135:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:0.0:1787827716.853196:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827713/real 1787827716] req@ffff907d9114b840 x1874670011608064/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827716.853231:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011608064/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827716.853248:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827716.853256:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827716.853263:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011608064:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827716.853275:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827713/real 1787827716] req@ffff907d9114a580 x1874670011608192/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827716.853291:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011608192/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827716.853301:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827716.853305:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827716.853310:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011608192:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827716.853318:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827713/real 1787827716] req@ffff907d91149a40 x1874670011608320/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827716.853339:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011608320/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827716.853347:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827716.853351:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827716.853355:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011608320:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827716.853362:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827713/real 1787827716] req@ffff907d9114ad00 x1874670011607936/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827716.853377:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011607936/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827716.853385:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827716.853389:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827716.853392:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011607936:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827716.980675:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827716.980763:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827718.004705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827718.004789:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827718.900714:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827718.900725:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827718.900731:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827718.900743:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3122 00000100:00080000:2.0:1787827718.900757:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827718.900764:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827718.900805:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011608448/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827718.900844:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827718.900847:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827718.900849:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827718.900852:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3122 00000100:00080000:2.0:1787827718.900854:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827718.900857:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827718.900867:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827718.900876:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011608576/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827718.900883:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827718.900886:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827718.900888:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827718.900890:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3122 00000100:00080000:2.0:1787827718.900892:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827718.900895:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827718.900898:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827718.900904:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011608704/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827718.900911:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827718.900915:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827718.900916:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827718.900919:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3122 00000100:00080000:2.0:1787827718.900922:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827718.900926:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827718.900930:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827718.900940:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011608448:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827718.900941:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011608832/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827718.901032:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827718.901111:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827718/real 1787827718] req@ffff907d91148b40 x1874670011608448/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827718.901134:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011608448/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827718.901147:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827718.901152:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827718.901157:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011608448:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827718.901166:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011608576:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827718.901196:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827718.901212:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827718/real 1787827718] req@ffff907d911492c0 x1874670011608576/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827718.901226:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011608576/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827718.901236:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827718.901240:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827718.901244:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011608576:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827718.901251:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011608704:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827718.901278:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827718.901291:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827718/real 1787827718] req@ffff907d91148f00 x1874670011608704/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827718.901305:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011608704/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827718.901316:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827718.901320:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827718.901324:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011608704:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827718.901357:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011608832:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827718.901383:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827718.901396:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827718/real 1787827718] req@ffff907d9114a940 x1874670011608832/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827718.901411:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011608832/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827718.901422:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827718.901445:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827718.901449:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011608832:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827719.028666:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827719.028761:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827720.052741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827720.052798:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827721.076678:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827721.076800:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827722.100700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827722.100828:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827723.124670:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827723.124812:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827724.020679:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827724.020690:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827724.020694:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827724.020703:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3127 00000100:00080000:2.0:1787827724.020713:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827724.020720:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827724.020773:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011608960/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827724.020794:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827724.020796:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827724.020798:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827724.020801:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3127 00000100:00080000:2.0:1787827724.020803:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827724.020805:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827724.020813:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827724.020822:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011609088/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827724.020829:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827724.020831:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827724.020832:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827724.020841:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3127 00000100:00080000:2.0:1787827724.020843:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827724.020846:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827724.020849:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827724.020856:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011609216/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827724.020865:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827724.020869:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827724.020870:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827724.020873:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3127 00000100:00080000:2.0:1787827724.020875:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827724.020878:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827724.020881:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827724.020889:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011609344/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827724.020896:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011608960:192.168.203.104@tcp:250:kworker.0 00000800:00000100:0.0:1787827724.021005:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827724.021067:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827724/real 1787827724] req@ffff907d9114a940 x1874670011608960/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827724.021086:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011608960/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827724.021098:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827724.021102:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827724.021106:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011608960:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827724.021114:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011609088:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827724.021135:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827724.021145:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827724/real 1787827724] req@ffff907d91148f00 x1874670011609088/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827724.021155:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011609088/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827724.021168:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827724.021171:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827724.021174:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011609088:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827724.021179:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011609216:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827724.021193:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827724.021216:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827724/real 1787827724] req@ffff907d911492c0 x1874670011609216/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827724.021225:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011609216/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827724.021230:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827724.021232:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827724.021234:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011609216:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827724.021259:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011609344:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827724.021288:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827724.021297:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827724/real 1787827724] req@ffff907d91148b40 x1874670011609344/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827724.021306:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011609344/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827724.021313:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827724.021315:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827724.021319:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011609344:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827724.148725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827724.148828:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827725.172724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827725.172792:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827726.196726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827726.196830:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827727.220768:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827727.220931:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827728.244722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827728.244835:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827729.140802:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827729.140817:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827729.140825:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827729.140841:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3132 00000100:00080000:2.0:1787827729.140859:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827729.140884:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827729.140939:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011609472/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827729.140985:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827729.140992:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827729.140998:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827729.141006:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3132 00000100:00080000:2.0:1787827729.141016:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827729.141024:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827729.141041:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827729.141067:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011609600/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827729.141087:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011609472:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827729.141092:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827729.141106:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827729.141109:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827729.141119:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3132 00000100:00080000:2.0:1787827729.141126:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827729.141133:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827729.141144:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827729.141165:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011609728/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827729.141183:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827729.141189:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827729.141196:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787827729.141198:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787827729.141212:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3132 00000100:00080000:2.0:1787827729.141220:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827729.141229:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827729.141241:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827729.141266:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827729/real 1787827729] req@ffff907d9114ad00 x1874670011609472/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827729.141269:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011609856/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827729.141314:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011609472/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827729.141328:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827729.141333:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827729.141337:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011609472:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827729.141388:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011609600:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827729.141417:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827729.141433:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827729/real 1787827729] req@ffff907d91149a40 x1874670011609600/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827729.141445:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011609600/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827729.141457:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827729.141462:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827729.141466:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011609600:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827729.141476:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011609728:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827729.141498:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827729.141508:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827729/real 1787827729] req@ffff907d9114a580 x1874670011609728/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827729.141518:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011609728/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827729.141527:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827729.141530:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827729.141532:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011609728:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827729.141545:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011609856:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827729.141561:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827729.141571:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827729/real 1787827729] req@ffff907d9114b840 x1874670011609856/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827729.141584:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011609856/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827729.141592:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827729.141596:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827729.141600:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011609856:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827729.268789:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827729.268869:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827730.292733:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827730.292781:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787827731.316699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827731.316825:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827732.340766:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827732.340774:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827733.364698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827733.364747:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827734.260703:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827734.260715:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827734.260721:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827734.260734:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3138 00000100:00080000:1.0:1787827734.260748:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827734.260756:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827734.260802:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011609984/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827734.260849:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827734.260853:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827734.260856:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827734.260860:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3138 00000100:00080000:1.0:1787827734.260864:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827734.260869:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827734.260881:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827734.260897:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011610112/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827734.260910:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827734.260914:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827734.260916:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827734.260924:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3138 00000100:00100000:0.0:1787827734.260927:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011609984:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827734.260928:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827734.260939:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827734.260946:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827734.260961:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011610240/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827734.260979:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827734.260983:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827734.260985:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827734.260989:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3138 00000100:00080000:1.0:1787827734.261004:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827734.261019:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827734.261024:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787827734.261035:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827734.261039:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011610368/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827734.261107:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827734/real 1787827734] req@ffff907d908a2d00 x1874670011609984/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827734.261138:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011609984/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827734.261183:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827734.261191:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827734.261199:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011609984:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827734.261257:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011610112:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827734.261277:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827734.261288:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827734/real 1787827734] req@ffff907d908a1e00 x1874670011610112/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827734.261297:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011610112/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827734.261303:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827734.261306:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827734.261308:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011610112:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827734.261312:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011610240:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827734.261341:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827734.261375:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827734/real 1787827734] req@ffff907d908a0000 x1874670011610240/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827734.261388:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011610240/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827734.261397:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827734.261401:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827734.261404:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011610240:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827734.261411:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011610368:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827734.261430:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827734.261439:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827734/real 1787827734] req@ffff907d908a2940 x1874670011610368/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827734.261450:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011610368/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827734.261459:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827734.261461:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827734.261464:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011610368:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827734.388720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827734.388843:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827735.412699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827735.412771:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827736.436662:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827736.436773:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827737.460717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827737.460777:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827738.484760:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827738.484886:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827739.380729:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827739.380743:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827739.380748:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827739.380762:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3143 00000100:00080000:3.0:1787827739.380777:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827739.380785:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827739.380827:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011610496/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827739.380855:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827739.380858:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827739.380860:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827739.380863:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3143 00000100:00080000:3.0:1787827739.380866:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827739.380868:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827739.380878:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827739.380889:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011610624/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827739.380898:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827739.380900:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827739.380902:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827739.380907:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3143 00000100:00100000:0.0:1787827739.380909:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011610496:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827739.380910:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827739.380915:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827739.380922:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827739.380930:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011610752/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827739.380938:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827739.380940:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827739.380942:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827739.380944:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3143 00000100:00080000:3.0:1787827739.380946:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827739.380948:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827739.380951:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827739.380960:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011610880/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827739.381000:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827739.381057:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827739/real 1787827739] req@ffff907da562f480 x1874670011610496/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827739.381072:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011610496/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827739.381088:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827739.381092:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827739.381096:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011610496:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827739.381121:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011610624:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827739.381140:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827739.381149:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827739/real 1787827739] req@ffff907da562c000 x1874670011610624/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827739.381158:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011610624/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827739.381164:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827739.381166:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827739.381169:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011610624:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827739.381173:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011610752:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827739.381184:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827739.381191:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827739/real 1787827739] req@ffff907da562e580 x1874670011610752/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827739.381199:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011610752/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827739.381204:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827739.381206:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827739.381207:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011610752:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827739.381211:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011610880:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827739.381220:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827739.381225:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827739/real 1787827739] req@ffff907da562d2c0 x1874670011610880/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827739.381232:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011610880/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827739.381242:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827739.381244:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827739.381246:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011610880:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827739.508673:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827739.508747:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827740.532705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827740.532794:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827741.556727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827741.556811:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827742.580739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827742.580785:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827743.604704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827743.604806:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827744.500791:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827744.500810:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827744.500817:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827744.500835:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3148 00000100:00080000:2.0:1787827744.500855:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827744.500865:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827744.500922:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011611008/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827744.500962:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827744.500966:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827744.500969:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827744.500973:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3148 00000100:00080000:2.0:1787827744.500977:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827744.500982:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827744.500996:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827744.501028:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011611136/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827744.501041:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827744.501047:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827744.501052:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787827744.501056:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011611008:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827744.501057:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3148 00000100:00080000:2.0:1787827744.501066:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827744.501076:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827744.501102:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827744.501117:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011611264/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827744.501128:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827744.501132:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827744.501135:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827744.501139:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3148 00000100:00080000:2.0:1787827744.501143:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827744.501147:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827744.501152:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827744.501165:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011611392/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827744.501188:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827744.501265:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827744/real 1787827744] req@ffff907d91149680 x1874670011611008/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827744.501283:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011611008/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827744.501293:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827744.501297:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827744.501301:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011611008:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827744.501324:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011611136:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827744.501347:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827744.501358:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827744/real 1787827744] req@ffff907d9114bc00 x1874670011611136/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827744.501367:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011611136/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827744.501374:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827744.501377:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827744.501379:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011611136:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827744.501386:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011611264:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827744.501401:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827744.501408:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827744/real 1787827744] req@ffff907d911483c0 x1874670011611264/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827744.501425:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011611264/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827744.501431:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827744.501434:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827744.501436:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011611264:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827744.501440:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011611392:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827744.501452:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827744.501458:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827744/real 1787827744] req@ffff907d9114b480 x1874670011611392/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827744.501467:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011611392/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827744.501473:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827744.501475:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827744.501477:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011611392:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827744.628770:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827744.628903:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827745.652738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827745.652784:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827746.676726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827746.676815:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827747.700696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827747.700754:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827748.724760:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827748.724828:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827749.620717:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827749.620728:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827749.620733:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827749.620745:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3153 00000100:00080000:2.0:1787827749.620757:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827749.620764:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827749.620838:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011611520/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827749.620871:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827749.620874:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827749.620877:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827749.620880:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3153 00000100:00080000:2.0:1787827749.620883:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827749.620886:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827749.620897:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827749.620909:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011611648/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827749.620918:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827749.620936:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827749.620937:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827749.620940:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3153 00000100:00080000:2.0:1787827749.620943:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827749.620946:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827749.620949:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827749.620960:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011611520:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827749.620960:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011611776/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827749.620975:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827749.620978:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827749.620979:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827749.620982:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3153 00000100:00080000:2.0:1787827749.620983:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827749.621007:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827749.621012:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827749.621024:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011611904/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827749.621067:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827749.621119:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827749/real 1787827749] req@ffff907d9114b480 x1874670011611520/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827749.621137:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011611520/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827749.621147:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827749.621151:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827749.621154:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011611520:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827749.621161:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011611648:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827749.621179:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827749.621188:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827749/real 1787827749] req@ffff907d911483c0 x1874670011611648/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827749.621197:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011611648/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827749.621205:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827749.621207:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827749.621209:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011611648:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827749.621249:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011611776:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827749.621265:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827749.621274:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827749/real 1787827749] req@ffff907d9114bc00 x1874670011611776/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827749.621283:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011611776/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827749.621294:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827749.621297:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827749.621299:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011611776:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827749.621303:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011611904:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827749.621313:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827749.621320:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827749/real 1787827749] req@ffff907d91149680 x1874670011611904/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827749.621327:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011611904/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827749.621334:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827749.621336:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827749.621338:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011611904:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827749.748730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827749.748770:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827750.772728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827750.772808:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827751.796690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827751.796739:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827752.820708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827752.820766:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827753.844684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827753.844740:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827754.740728:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827754.740736:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827754.740739:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827754.740748:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3158 00000100:00080000:3.0:1787827754.740757:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827754.740763:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827754.740796:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011612032/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827754.740822:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827754.740826:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827754.740828:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827754.740833:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3158 00000100:00080000:3.0:1787827754.740836:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827754.740841:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827754.740851:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827754.740867:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011612160/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827754.740880:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787827754.740883:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011612032:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827754.740884:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827754.740887:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827754.740894:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3158 00000100:00080000:3.0:1787827754.740897:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827754.740900:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827754.740904:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827754.740914:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011612288/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827754.740921:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827754.740924:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827754.740925:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827754.740927:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3158 00000100:00080000:3.0:1787827754.740929:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827754.740931:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827754.740934:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827754.740941:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011612416/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827754.740958:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827754.741005:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827754/real 1787827754] req@ffff907da562c3c0 x1874670011612032/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827754.741023:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011612032/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827754.741034:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827754.741040:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827754.741044:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011612032:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827754.741068:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011612160:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827754.741093:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827754.741107:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827754/real 1787827754] req@ffff907da562f0c0 x1874670011612160/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827754.741121:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011612160/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827754.741130:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827754.741133:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827754.741137:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011612160:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827754.741143:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011612288:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827754.741163:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827754.741172:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827754/real 1787827754] req@ffff907da562fc00 x1874670011612288/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827754.741196:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011612288/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827754.741205:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827754.741208:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827754.741212:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011612288:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827754.741219:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011612416:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827754.741240:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827754.741252:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827754/real 1787827754] req@ffff907da562e940 x1874670011612416/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827754.741266:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011612416/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827754.741274:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827754.741277:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827754.741280:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011612416:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827754.868701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827754.868777:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827755.892718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827755.892820:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827756.916720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827756.916765:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827757.940695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827757.940776:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827758.964771:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787827758.964846:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:1.0:1787827759.860708:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827759.860723:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827759.860726:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827759.860735:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3163 00000100:00080000:1.0:1787827759.860745:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827759.860751:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827759.860781:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011612544/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827759.860802:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827759.860805:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827759.860806:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827759.860809:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3163 00000100:00080000:1.0:1787827759.860812:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827759.860815:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827759.860823:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827759.860834:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011612672/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827759.860843:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827759.860845:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827759.860847:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827759.860850:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3163 00000100:00080000:1.0:1787827759.860853:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827759.860855:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827759.860858:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827759.860865:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011612800/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827759.860872:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827759.860876:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827759.860878:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787827759.860884:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011612544:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827759.860884:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3163 00000100:00080000:1.0:1787827759.860893:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827759.860906:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827759.860911:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827759.860919:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011612928/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827759.860975:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827759.861027:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827759/real 1787827759] req@ffff907d908a2580 x1874670011612544/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827759.861044:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011612544/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827759.861054:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827759.861058:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827759.861061:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011612544:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827759.861068:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011612672:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827759.861084:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827759.861093:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827759/real 1787827759] req@ffff907d908a21c0 x1874670011612672/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827759.861101:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011612672/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827759.861108:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827759.861110:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827759.861113:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011612672:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827759.861135:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011612800:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827759.861164:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827759.861172:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827759/real 1787827759] req@ffff907d908a0f00 x1874670011612800/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827759.861184:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011612800/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827759.861193:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827759.861197:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827759.861201:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011612800:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827759.861209:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011612928:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827759.861230:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827759.861242:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827759/real 1787827759] req@ffff907d908a12c0 x1874670011612928/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827759.861256:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011612928/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827759.861265:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827759.861268:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827759.861273:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011612928:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827759.988745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827759.988844:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827761.012731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827761.012747:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827762.036696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827762.036786:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827763.060733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827763.060781:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827764.084713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827764.084745:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827764.980747:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827764.980760:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827764.980765:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827764.980776:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3168 00000100:00080000:1.0:1787827764.980789:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827764.980796:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827764.980839:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011613056/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827764.980874:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827764.980877:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827764.980879:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827764.980881:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3168 00000100:00080000:1.0:1787827764.980885:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827764.980888:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827764.980900:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827764.980915:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011613184/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827764.980928:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827764.980931:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827764.980934:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827764.980940:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3168 00000100:00100000:3.0:1787827764.980941:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011613056:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827764.980946:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827764.980954:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827764.980960:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827764.980973:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011613312/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827764.980980:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827764.980983:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827764.980984:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827764.981004:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3168 00000100:00080000:1.0:1787827764.981007:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827764.981011:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827764.981015:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827764.981031:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011613440/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827764.981161:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827764.981399:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827764/real 1787827764] req@ffff907d908a3840 x1874670011613056/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827764.981427:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011613056/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827764.981442:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827764.981448:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827764.981453:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011613056:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827764.981479:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011613184:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827764.981510:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827764.981527:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827764/real 1787827764] req@ffff907d908a1680 x1874670011613184/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827764.981542:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011613184/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827764.981552:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827764.981556:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827764.981560:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011613184:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827764.981579:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011613312:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827764.981646:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827764.981663:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827764/real 1787827764] req@ffff907d908a12c0 x1874670011613312/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827764.981678:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011613312/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827764.981688:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827764.981693:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827764.981697:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011613312:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827764.981707:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011613440:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827764.981732:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827764.981746:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827764/real 1787827764] req@ffff907d908a0f00 x1874670011613440/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827764.981767:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011613440/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827764.981776:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827764.981780:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827764.981784:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011613440:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827765.108715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827765.108786:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827766.132746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827766.132795:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827767.156678:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827767.156723:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827768.180723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827768.180802:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827769.204721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827769.204749:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827770.100716:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827770.100727:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827770.100732:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827770.100743:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3173 00000100:00080000:1.0:1787827770.100755:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827770.100763:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827770.100800:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011613568/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827770.100831:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827770.100836:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827770.100839:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827770.100843:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3173 00000100:00080000:1.0:1787827770.100847:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827770.100851:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827770.100862:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827770.100876:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011613696/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827770.100888:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827770.100892:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827770.100894:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827770.100899:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3173 00000100:00080000:1.0:1787827770.100903:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827770.100907:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827770.100916:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011613568:192.168.203.104@tcp:250:kworker.0 00000008:00100000:1.0:1787827770.100927:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827770.100943:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011613824/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827770.100954:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827770.100958:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827770.100960:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827770.100965:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3173 00000100:00080000:1.0:1787827770.100969:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827770.100973:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827770.100978:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787827770.100991:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827770.100993:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011613952/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827770.101045:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827770/real 1787827770] req@ffff907d908a21c0 x1874670011613568/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827770.101089:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011613568/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827770.101101:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827770.101106:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827770.101111:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011613568:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827770.101119:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011613696:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827770.101141:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827770.101161:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827770/real 1787827770] req@ffff907d908a2580 x1874670011613696/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827770.101177:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011613696/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827770.101186:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827770.101189:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827770.101192:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011613696:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827770.101214:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011613824:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827770.101231:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827770.101241:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827770/real 1787827770] req@ffff907d908a2940 x1874670011613824/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827770.101259:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011613824/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827770.101268:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827770.101271:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827770.101274:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011613824:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827770.101279:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011613952:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827770.101292:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827770.101301:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827770/real 1787827770] req@ffff907d908a0000 x1874670011613952/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827770.101328:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011613952/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827770.101338:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827770.101341:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827770.101344:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011613952:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827770.228704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827770.228774:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827771.252714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827771.252747:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827772.276707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827772.276758:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827773.300704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827773.300736:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827774.324745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827774.324846:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827775.220840:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827775.220851:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827775.220856:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827775.220869:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3178 00000100:00080000:2.0:1787827775.220884:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827775.220892:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827775.220938:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011614080/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827775.220981:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827775.220985:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827775.220987:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827775.220990:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3178 00000100:00080000:2.0:1787827775.220993:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827775.220996:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827775.221008:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827775.221020:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011614208/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827775.221030:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827775.221052:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827775.221055:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827775.221059:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3178 00000100:00080000:2.0:1787827775.221073:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827775.221076:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827775.221082:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827775.221093:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011614080:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827775.221095:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011614336/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827775.221123:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827775.221126:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827775.221129:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827775.221135:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3178 00000100:00080000:2.0:1787827775.221138:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827775.221141:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827775.221146:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827775.221159:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011614464/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787827775.221182:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827775.221230:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827775/real 1787827775] req@ffff907d9114b840 x1874670011614080/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827775.221248:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011614080/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827775.221258:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827775.221261:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827775.221264:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011614080:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827775.221270:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011614208:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827775.221286:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827775.221294:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827775/real 1787827775] req@ffff907d9114a580 x1874670011614208/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827775.221302:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011614208/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827775.221307:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827775.221310:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827775.221312:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011614208:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827775.221331:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011614336:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827775.221343:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827775.221349:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827775/real 1787827775] req@ffff907d91149a40 x1874670011614336/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827775.221356:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011614336/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827775.221366:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827775.221368:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827775.221371:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011614336:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827775.221374:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011614464:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827775.221383:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827775.221389:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827775/real 1787827775] req@ffff907d9114ad00 x1874670011614464/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827775.221395:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011614464/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827775.221401:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827775.221402:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827775.221404:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011614464:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827775.348695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827775.348866:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827776.372703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827776.372762:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827777.396719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827777.396777:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827778.420699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827778.420728:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827779.444693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827779.444785:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827780.340704:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827780.340715:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827780.340720:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827780.340731:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3184 00000100:00080000:3.0:1787827780.340742:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827780.340750:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827780.340792:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011614592/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827780.340822:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827780.340829:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827780.340831:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827780.340834:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3184 00000100:00080000:3.0:1787827780.340837:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827780.340840:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827780.340851:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827780.340862:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011614720/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827780.340871:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827780.340876:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787827780.340877:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011614592:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827780.340879:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827780.340886:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3184 00000100:00080000:3.0:1787827780.340889:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827780.340892:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827780.340895:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827780.340906:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011614848/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827780.340913:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827780.340916:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827780.340917:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827780.340921:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3184 00000100:00080000:3.0:1787827780.340925:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827780.340928:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827780.340931:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827780.340941:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011614976/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827780.341018:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011614720:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827780.341064:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011614848:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827780.341082:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011614976:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827780.468713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827780.468903:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827781.492706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827781.492768:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827782.516742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827782.516787:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787827783.412870:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787827783.412916:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787827783.412933:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787827783.412971:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787827783.413000:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787827783.413010:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:0.0:1787827783.413052:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827780/real 1787827783] req@ffff907da562da40 x1874670011614720/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827783.413093:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011614720/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827783.413109:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827783.413116:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827783.413121:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011614720:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827783.413131:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827780/real 1787827783] req@ffff907da562f840 x1874670011614848/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827783.413146:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011614848/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827783.413157:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827783.413163:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827783.413167:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011614848:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827783.413175:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827780/real 1787827783] req@ffff907da562e1c0 x1874670011614976/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827783.413188:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011614976/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827783.413197:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827783.413201:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827783.413204:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011614976:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827783.413211:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827780/real 1787827783] req@ffff907da562cb40 x1874670011614592/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827783.413225:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011614592/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827783.413234:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827783.413237:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827783.413241:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011614592:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827783.540675:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827783.540786:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827784.564727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827784.564836:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827785.460685:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827785.460696:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827785.460700:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827785.460711:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3189 00000100:00080000:3.0:1787827785.460723:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827785.460732:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827785.460771:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011615104/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827785.460799:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827785.460802:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827785.460804:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827785.460807:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3189 00000100:00080000:3.0:1787827785.460811:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827785.460814:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827785.460824:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827785.460840:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011615232/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827785.460854:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827785.460858:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827785.460860:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827785.460864:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3189 00000100:00080000:3.0:1787827785.460868:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827785.460873:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827785.460880:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827785.460902:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011615360/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827785.460903:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011615104:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787827785.460927:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827785.460932:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827785.460935:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827785.460941:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3189 00000100:00080000:3.0:1787827785.460947:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827785.460952:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827785.460958:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827785.460993:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011615488/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827785.461028:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827785.461104:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827785/real 1787827785] req@ffff907da562de00 x1874670011615104/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827785.461134:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011615104/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827785.461149:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827785.461155:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827785.461161:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011615104:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827785.461171:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011615232:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827785.461200:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827785.461216:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827785/real 1787827785] req@ffff907da562cb40 x1874670011615232/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827785.461229:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011615232/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827785.461238:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827785.461249:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827785.461253:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011615232:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827785.461289:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011615360:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827785.461314:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827785.461328:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827785/real 1787827785] req@ffff907da562e1c0 x1874670011615360/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827785.461340:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011615360/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827785.461349:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827785.461353:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827785.461357:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011615360:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827785.461364:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011615488:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827785.461386:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827785.461398:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827785/real 1787827785] req@ffff907da562f840 x1874670011615488/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827785.461413:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011615488/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827785.461422:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827785.461426:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827785.461430:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011615488:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827785.588720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827785.588777:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827786.612728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827786.612876:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827787.636705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827787.636740:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827788.660722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827788.660747:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827789.684746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827789.684809:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827790.580773:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827790.580784:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827790.580788:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827790.580800:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3194 00000100:00080000:1.0:1787827790.580811:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827790.580818:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827790.580861:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011615616/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827790.580887:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827790.580891:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827790.580892:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827790.580895:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3194 00000100:00080000:1.0:1787827790.580897:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827790.580899:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827790.580910:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827790.580920:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011615744/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827790.580946:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827790.580949:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827790.580952:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787827790.580955:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011615616:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827790.580957:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3194 00000100:00080000:1.0:1787827790.580962:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827790.580967:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827790.580972:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827790.580983:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011615872/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827790.580990:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827790.580993:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827790.580995:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827790.580997:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3194 00000100:00080000:1.0:1787827790.580999:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827790.581001:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827790.581004:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827790.581010:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011616000/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827790.581041:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827790.581094:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827790/real 1787827790] req@ffff907d908a1e00 x1874670011615616/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827790.581115:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011615616/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827790.581129:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827790.581134:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827790.581139:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011615616:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827790.581168:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011615744:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827790.581193:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827790.581206:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827790/real 1787827790] req@ffff907d908a2d00 x1874670011615744/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827790.581219:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011615744/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827790.581229:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827790.581233:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827790.581236:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011615744:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827790.581243:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011615872:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827790.581264:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827790.581276:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827790/real 1787827790] req@ffff907d908a0000 x1874670011615872/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827790.581289:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011615872/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827790.581298:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827790.581302:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827790.581305:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011615872:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827790.581312:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011616000:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827790.581342:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827790.581354:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827790/real 1787827790] req@ffff907d908a2940 x1874670011616000/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827790.581366:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011616000/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827790.581377:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827790.581380:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827790.581384:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011616000:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827790.708727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827790.708765:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827791.732708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827791.732819:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827792.756715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827792.756735:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827793.780732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827793.780793:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827794.804751:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827794.804831:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827795.700724:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827795.700738:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827795.700745:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827795.700760:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3199 00000100:00080000:2.0:1787827795.700775:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827795.700784:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827795.700842:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011616128/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827795.700876:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827795.700882:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827795.700884:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827795.700906:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3199 00000100:00080000:2.0:1787827795.700911:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827795.700917:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827795.700931:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827795.700947:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011616256/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827795.700974:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787827795.700978:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011616128:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827795.700979:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827795.700985:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827795.701001:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3199 00000100:00080000:2.0:1787827795.701006:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827795.701012:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827795.701020:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827795.701036:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011616384/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827795.701050:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827795.701055:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827795.701057:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787827795.701065:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787827795.701068:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3199 00000100:00080000:2.0:1787827795.701084:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827795.701089:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827795.701096:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827795.701111:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011616512/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827795.701152:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827795/real 1787827795] req@ffff907d91148b40 x1874670011616128/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827795.701171:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011616128/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827795.701181:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827795.701185:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827795.701188:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011616128:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827795.701213:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011616256:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827795.701256:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827795.701267:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827795/real 1787827795] req@ffff907d911492c0 x1874670011616256/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827795.701277:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011616256/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827795.701283:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827795.701286:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827795.701301:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011616256:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827795.701305:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011616384:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827795.701328:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827795.701336:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827795/real 1787827795] req@ffff907d91148f00 x1874670011616384/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827795.701345:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011616384/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827795.701351:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827795.701365:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827795.701368:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011616384:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827795.701372:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011616512:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827795.701384:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827795.701390:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827795/real 1787827795] req@ffff907d9114a940 x1874670011616512/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827795.701398:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011616512/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827795.701404:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827795.701406:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827795.701408:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011616512:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827795.828757:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827795.828811:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827796.852745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827796.852874:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827797.876716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827797.876907:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827798.900703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827798.900749:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827799.924684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827799.924756:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827800.820840:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827800.820874:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827800.820882:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827800.820899:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3204 00000100:00080000:2.0:1787827800.820918:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827800.820933:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827800.820993:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011616640/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827800.821033:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827800.821039:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827800.821042:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827800.821050:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3204 00000100:00080000:2.0:1787827800.821057:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827800.821062:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827800.821076:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827800.821095:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011616768/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827800.821113:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827800.821121:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787827800.821123:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011616640:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827800.821124:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827800.821137:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3204 00000100:00080000:2.0:1787827800.821146:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827800.821153:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827800.821162:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827800.821177:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011616896/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827800.821191:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827800.821196:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827800.821200:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827800.821204:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3204 00000100:00080000:2.0:1787827800.821212:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827800.821220:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827800.821225:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787827800.821234:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827800.821242:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011617024/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827800.821313:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827800/real 1787827800] req@ffff907d9114a940 x1874670011616640/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827800.821334:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011616640/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827800.821346:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827800.821350:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827800.821354:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011616640:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827800.821386:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011616768:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827800.821409:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827800.821423:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827800/real 1787827800] req@ffff907d91148f00 x1874670011616768/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827800.821437:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011616768/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827800.821447:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827800.821451:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827800.821455:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011616768:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827800.821462:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011616896:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827800.821478:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827800.821487:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827800/real 1787827800] req@ffff907d911492c0 x1874670011616896/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827800.821503:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011616896/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827800.821509:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827800.821511:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827800.821513:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011616896:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827800.821518:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011617024:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827800.821530:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827800.821537:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827800/real 1787827800] req@ffff907d91148b40 x1874670011617024/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827800.821546:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011617024/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827800.821553:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827800.821556:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827800.821558:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011617024:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827800.948778:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827800.948928:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827801.972701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827801.972785:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827802.996732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827802.996820:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827804.020679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827804.020746:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827805.044791:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827805.044887:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827805.940793:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827805.940805:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827805.940811:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827805.940843:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3209 00000100:00080000:2.0:1787827805.940859:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827805.940882:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827805.940939:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011617152/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827805.940995:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827805.941000:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827805.941003:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827805.941008:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3209 00000100:00080000:2.0:1787827805.941013:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827805.941031:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827805.941052:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827805.941073:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011617280/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827805.941085:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827805.941089:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827805.941092:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827805.941096:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3209 00000100:00080000:2.0:1787827805.941100:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827805.941107:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827805.941116:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827805.941125:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011617152:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827805.941132:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011617408/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827805.941163:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827805.941166:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827805.941168:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827805.941174:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3209 00000100:00080000:2.0:1787827805.941178:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827805.941182:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827805.941187:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827805.941198:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011617536/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827805.941292:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827805.941393:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827805/real 1787827805] req@ffff907d9114ad00 x1874670011617152/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827805.941456:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011617152/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827805.941497:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827805.941507:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827805.941517:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011617152:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827805.941534:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011617280:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827805.941598:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827805.941693:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827805/real 1787827805] req@ffff907d91149a40 x1874670011617280/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827805.941722:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011617280/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827805.941739:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827805.941764:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827805.941772:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011617280:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827805.941865:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011617408:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827805.941907:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827805.941927:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827805/real 1787827805] req@ffff907d9114a580 x1874670011617408/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827805.941963:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011617408/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827805.941979:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827805.941986:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827805.941992:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011617408:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827805.942006:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011617536:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827805.942052:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827805.942067:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827805/real 1787827805] req@ffff907d9114b840 x1874670011617536/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827805.942086:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011617536/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827805.942099:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827805.942105:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827805.942109:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011617536:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827806.068781:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827806.069000:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827807.092748:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827807.092928:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827808.116745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827808.116844:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827809.140688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827809.140942:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827810.164664:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827810.164762:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827811.060736:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827811.060745:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827811.060748:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827811.060757:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3214 00000100:00080000:1.0:1787827811.060767:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827811.060773:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827811.060840:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011617664/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827811.060868:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827811.060871:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827811.060873:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827811.060876:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3214 00000100:00080000:1.0:1787827811.060878:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827811.060881:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827811.060890:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827811.060902:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011617792/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827811.060911:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827811.060915:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787827811.060916:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011617664:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827811.060917:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827811.060925:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3214 00000100:00080000:1.0:1787827811.060928:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827811.060930:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827811.060934:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827811.060942:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011617920/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827811.060948:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827811.060951:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827811.060952:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827811.060955:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3214 00000100:00080000:1.0:1787827811.060957:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827811.060959:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827811.060962:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827811.060969:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011618048/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827811.060995:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827811.061039:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827811/real 1787827811] req@ffff907d908a2580 x1874670011617664/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827811.061055:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011617664/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827811.061063:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827811.061066:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827811.061069:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011617664:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827811.061089:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011617792:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827811.061122:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827811.061132:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827811/real 1787827811] req@ffff907d908a21c0 x1874670011617792/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827811.061140:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011617792/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827811.061146:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827811.061148:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827811.061151:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011617792:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827811.061154:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011617920:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827811.061166:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827811.061173:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827811/real 1787827811] req@ffff907d908a0f00 x1874670011617920/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827811.061180:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011617920/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827811.061186:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827811.061187:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827811.061189:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011617920:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827811.061193:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011618048:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827811.061203:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827811.061208:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827811/real 1787827811] req@ffff907d908a12c0 x1874670011618048/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827811.061216:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011618048/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827811.061221:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827811.061223:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827811.061226:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011618048:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827811.188690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827811.188739:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827812.212707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827812.212793:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827813.236848:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827813.237154:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827814.260734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827814.260849:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827815.284735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827815.284871:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827816.180736:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827816.180767:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827816.180773:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827816.180785:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3219 00000100:00080000:1.0:1787827816.180798:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827816.180807:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827816.180851:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011618176/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827816.180885:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827816.180890:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827816.180892:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827816.180897:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3219 00000100:00080000:1.0:1787827816.180902:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827816.180907:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827816.180919:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827816.180935:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011618304/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827816.180952:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011618176:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827816.180953:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827816.180966:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827816.180971:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827816.180978:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3219 00000100:00080000:1.0:1787827816.180982:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827816.180985:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827816.180991:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827816.181006:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011618432/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827816.181017:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827816.181021:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827816.181023:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827816.181029:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3219 00000100:00080000:1.0:1787827816.181044:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827816.181049:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827816.181054:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787827816.181069:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827816.181071:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011618560/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827816.181135:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827816/real 1787827816] req@ffff907d908a1680 x1874670011618176/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827816.181169:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011618176/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827816.181186:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827816.181202:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827816.181207:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011618176:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827816.181237:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011618304:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827816.181271:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827816.181287:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827816/real 1787827816] req@ffff907d908a3840 x1874670011618304/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827816.181302:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011618304/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827816.181312:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827816.181316:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827816.181321:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011618304:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827816.181329:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011618432:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827816.181349:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827816.181359:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827816/real 1787827816] req@ffff907d908a12c0 x1874670011618432/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827816.181373:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011618432/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827816.181382:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827816.181385:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827816.181389:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011618432:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827816.181395:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011618560:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827816.181412:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827816.181421:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827816/real 1787827816] req@ffff907d908a0f00 x1874670011618560/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827816.181434:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011618560/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827816.181443:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827816.181447:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827816.181450:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011618560:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827816.308723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827816.308823:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827817.332712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827817.332872:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827818.356795:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827818.356911:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827819.380749:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827819.380825:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827820.404853:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827820.404980:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827821.300703:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827821.300726:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827821.300730:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827821.300739:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3225 00000100:00080000:1.0:1787827821.300749:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827821.300755:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827821.300787:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011618688/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827821.300813:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827821.300817:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827821.300819:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827821.300823:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3225 00000100:00080000:1.0:1787827821.300827:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827821.300832:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827821.300842:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827821.300857:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011618816/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827821.300871:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827821.300876:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827821.300878:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827821.300882:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3225 00000100:00080000:1.0:1787827821.300887:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827821.300891:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827821.300896:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827821.300908:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011618944/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827821.300918:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827821.300921:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827821.300925:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827821.300934:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3225 00000100:00100000:0.0:1787827821.300935:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011618688:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827821.300939:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827821.300954:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827821.300959:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827821.300967:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011619072/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827821.301056:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827821.301134:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827821/real 1787827821] req@ffff907d908a21c0 x1874670011618688/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827821.301162:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011618688/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827821.301178:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827821.301184:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827821.301190:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011618688:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827821.301200:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011618816:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827821.301247:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827821.301264:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827821/real 1787827821] req@ffff907d908a2580 x1874670011618816/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827821.301280:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011618816/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827821.301291:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827821.301295:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827821.301299:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011618816:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827821.301333:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011618944:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827821.301357:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827821.301369:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827821/real 1787827821] req@ffff907d908a2940 x1874670011618944/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827821.301384:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011618944/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827821.301403:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827821.301407:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827821.301411:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011618944:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827821.301418:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011619072:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827821.301442:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827821.301462:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827821/real 1787827821] req@ffff907d908a0000 x1874670011619072/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827821.301485:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011619072/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827821.301502:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827821.301509:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827821.301516:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011619072:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827821.428779:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827821.428885:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827822.452788:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827822.452935:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827823.476676:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827823.476775:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827824.500800:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827824.500924:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827825.524712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827825.524870:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827826.420797:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827826.420811:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827826.420817:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827826.420831:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3230 00000100:00080000:2.0:1787827826.420846:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827826.420855:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827826.420923:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011619200/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827826.420965:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827826.420969:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827826.420971:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827826.420975:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3230 00000100:00080000:2.0:1787827826.420979:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827826.420982:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827826.420995:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827826.421008:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011619328/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827826.421019:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827826.421023:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827826.421025:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827826.421029:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3230 00000100:00080000:2.0:1787827826.421034:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827826.421037:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827826.421042:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827826.421053:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011619456/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827826.421061:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827826.421064:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827826.421084:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827826.421090:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3230 00000100:00080000:2.0:1787827826.421095:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827826.421099:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827826.421106:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827826.421116:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011619200:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827826.421123:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011619584/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827826.421284:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827826.421372:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827826/real 1787827826] req@ffff907d91149680 x1874670011619200/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827826.421406:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011619200/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827826.421421:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827826.421427:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827826.421434:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011619200:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827826.421445:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011619328:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827826.421504:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827826.421533:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827826/real 1787827826] req@ffff907d9114bc00 x1874670011619328/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827826.421561:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011619328/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827826.421584:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827826.421588:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827826.421593:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011619328:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827826.421603:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011619456:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827826.421732:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827826.421751:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827826/real 1787827826] req@ffff907d911483c0 x1874670011619456/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827826.421767:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011619456/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827826.421778:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827826.421782:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827826.421786:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011619456:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827826.421820:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011619584:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827826.421843:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827826.421857:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827826/real 1787827826] req@ffff907d9114b480 x1874670011619584/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827826.421871:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011619584/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827826.421883:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827826.421889:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827826.421895:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011619584:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827826.548769:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827826.548861:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827827.572751:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827827.572908:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827828.596838:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827828.596972:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827829.620783:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827829.620937:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827830.644800:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827830.644882:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827831.540719:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827831.540728:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827831.540731:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827831.540740:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3235 00000100:00080000:1.0:1787827831.540749:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827831.540755:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827831.540785:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011619712/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827831.540806:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827831.540809:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827831.540811:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827831.540813:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3235 00000100:00080000:1.0:1787827831.540816:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827831.540818:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827831.540826:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827831.540835:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011619840/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827831.540843:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827831.540846:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827831.540849:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787827831.540851:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011619712:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827831.540854:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3235 00000100:00080000:1.0:1787827831.540859:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827831.540861:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827831.540864:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827831.540873:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011619968/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827831.540879:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827831.540882:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827831.540883:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827831.540886:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3235 00000100:00080000:1.0:1787827831.540888:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827831.540890:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827831.540893:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827831.540901:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011620096/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827831.540920:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827831.540957:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827831/real 1787827831] req@ffff907d908a2d00 x1874670011619712/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827831.540973:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011619712/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827831.540981:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827831.540984:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827831.540987:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011619712:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827831.541005:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011619840:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827831.541021:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827831.541030:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827831/real 1787827831] req@ffff907d908a1e00 x1874670011619840/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827831.541038:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011619840/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827831.541044:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827831.541053:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827831.541056:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011619840:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827831.541063:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011619968:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827831.541085:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827831.541098:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827831/real 1787827831] req@ffff907d908a0000 x1874670011619968/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827831.541111:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011619968/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827831.541120:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827831.541123:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827831.541127:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011619968:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827831.541134:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011620096:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827831.541156:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827831.541168:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827831/real 1787827831] req@ffff907d908a2940 x1874670011620096/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827831.541181:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011620096/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827831.541189:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827831.541192:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827831.541197:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011620096:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827831.668691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827831.668786:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827832.692732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827832.692825:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827833.716697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827833.716783:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827834.740790:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827834.740951:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827835.764716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827835.764760:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827836.660727:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827836.660735:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827836.660739:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827836.660748:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3240 00000100:00080000:1.0:1787827836.660759:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827836.660766:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827836.660817:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011620224/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827836.660843:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827836.660845:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827836.660847:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827836.660849:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3240 00000100:00080000:1.0:1787827836.660853:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827836.660856:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827836.660865:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827836.660879:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011620224:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827836.660879:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011620352/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827836.660894:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827836.660896:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827836.660897:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827836.660900:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3240 00000100:00080000:1.0:1787827836.660903:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827836.660906:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827836.660910:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827836.660922:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011620480/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827836.660932:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827836.660935:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827836.660937:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787827836.660942:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787827836.660946:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3240 00000100:00080000:1.0:1787827836.660952:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827836.660957:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827836.660963:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827836.660977:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011620608/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827836.660993:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827836/real 1787827836] req@ffff907d908a2580 x1874670011620224/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827836.661008:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011620224/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827836.661016:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827836.661021:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827836.661025:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011620224:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827836.661044:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011620352:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827836.661064:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827836.661075:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827836/real 1787827836] req@ffff907d908a21c0 x1874670011620352/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827836.661094:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011620352/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827836.661100:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827836.661102:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827836.661104:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011620352:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827836.661109:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011620480:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827836.661129:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827836.661138:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827836/real 1787827836] req@ffff907d908a0f00 x1874670011620480/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827836.661157:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011620480/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827836.661164:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827836.661167:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827836.661171:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011620480:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827836.661176:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011620608:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827836.661196:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827836.661209:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827836/real 1787827836] req@ffff907d908a12c0 x1874670011620608/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827836.661223:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011620608/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827836.661244:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827836.661249:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827836.661254:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011620608:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827836.788753:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827836.788818:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827837.812847:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787827837.812848:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:0.0:1787827838.836692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827838.836777:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827839.860745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827839.860801:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827840.884680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827840.884724:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827841.780713:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827841.780722:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827841.780726:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827841.780737:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3245 00000100:00080000:1.0:1787827841.780748:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827841.780755:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827841.780805:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011620736/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827841.780831:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827841.780838:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827841.780840:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827841.780843:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3245 00000100:00080000:1.0:1787827841.780846:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827841.780849:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827841.780863:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827841.780876:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011620864/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827841.780880:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011620736:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827841.780889:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827841.780893:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827841.780895:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827841.780899:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3245 00000100:00080000:1.0:1787827841.780903:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827841.780905:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827841.780913:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827841.780924:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011620992/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827841.780931:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827841.780934:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827841.780935:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827841.780938:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3245 00000100:00080000:1.0:1787827841.780941:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827841.780943:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827841.780947:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787827841.780952:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827841.780959:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011621120/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827841.780990:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827841/real 1787827841] req@ffff907d908a3840 x1874670011620736/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827841.781004:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011620736/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827841.781012:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827841.781021:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827841.781024:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011620736:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827841.781043:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011620864:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827841.781060:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827841.781068:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827841/real 1787827841] req@ffff907d908a1680 x1874670011620864/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827841.781077:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011620864/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827841.781082:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827841.781084:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827841.781086:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011620864:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827841.781096:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011620992:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827841.781109:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827841.781115:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827841/real 1787827841] req@ffff907d908a12c0 x1874670011620992/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827841.781123:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011620992/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827841.781133:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827841.781135:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827841.781137:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011620992:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827841.781141:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011621120:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827841.781151:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827841.781157:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827841/real 1787827841] req@ffff907d908a0f00 x1874670011621120/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827841.781164:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011621120/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827841.781169:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827841.781171:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827841.781172:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011621120:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827841.908697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827841.908754:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827842.932667:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827842.932703:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827843.956711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827843.956778:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827844.980713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827844.980752:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827846.004710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827846.004779:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827846.900750:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827846.900762:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827846.900766:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827846.900777:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3250 00000100:00080000:2.0:1787827846.900789:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827846.900797:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827846.900837:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011621248/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827846.900885:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827846.900890:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827846.900892:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827846.900897:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3250 00000100:00080000:2.0:1787827846.900901:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827846.900905:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827846.900915:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827846.900930:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011621376/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827846.900941:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827846.900945:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827846.900947:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827846.900950:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3250 00000100:00080000:2.0:1787827846.900954:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827846.900959:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827846.900965:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827846.900977:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011621504/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827846.900998:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827846.901002:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827846.901004:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827846.901009:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3250 00000100:00080000:2.0:1787827846.901013:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827846.901017:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827846.901023:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827846.901033:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011621248:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827846.901042:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011621632/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827846.901238:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011621376:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827846.901282:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011621504:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827846.901355:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011621632:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827847.028696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827847.028754:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827848.052723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827848.052775:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827849.076666:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827849.076793:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787827849.973097:0:2337:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:2.0:1787827849.973197:0:2337:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787827849.973224:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787827849.973290:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787827849.973306:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787827849.973320:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:0.0:1787827849.973388:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827846/real 1787827849] req@ffff907d91149680 x1874670011621632/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827849.973430:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011621632/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827849.973451:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827849.973459:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827849.973467:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011621632:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827849.973479:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827846/real 1787827849] req@ffff907d9114b480 x1874670011621248/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827849.973499:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011621248/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827849.973511:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827849.973538:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827849.973543:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011621248:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827849.973554:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827846/real 1787827849] req@ffff907d911483c0 x1874670011621376/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827849.973577:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011621376/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827849.973589:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827849.973595:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827849.973601:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011621376:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827849.973668:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827846/real 1787827849] req@ffff907d9114bc00 x1874670011621504/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827849.973691:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011621504/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827849.973703:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827849.973707:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827849.973711:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011621504:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827850.100659:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827850.100700:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827851.124689:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827851.124844:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827852.020814:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827852.020828:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827852.020851:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827852.020864:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3255 00000100:00080000:2.0:1787827852.020879:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827852.020888:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827852.020938:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011621760/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827852.020972:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827852.020975:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827852.020978:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827852.020983:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3255 00000100:00080000:2.0:1787827852.020986:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827852.020990:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827852.021002:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827852.021015:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011621888/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827852.021027:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827852.021030:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827852.021032:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827852.021036:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3255 00000100:00080000:2.0:1787827852.021039:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827852.021045:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827852.021051:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011621760:192.168.203.104@tcp:250:kworker.0 00000008:00100000:2.0:1787827852.021053:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827852.021071:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011622016/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827852.021083:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827852.021087:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827852.021089:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827852.021093:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3255 00000100:00080000:2.0:1787827852.021097:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827852.021101:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827852.021106:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827852.021121:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011622144/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827852.021151:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827852.021255:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827852/real 1787827852] req@ffff907d9114b840 x1874670011621760/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827852.021286:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011621760/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827852.021304:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827852.021310:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827852.021318:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011621760:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827852.021331:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011621888:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827852.021372:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827852.021392:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827852/real 1787827852] req@ffff907d9114a580 x1874670011621888/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827852.021409:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011621888/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827852.021421:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827852.021426:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827852.021431:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011621888:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827852.021472:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011622016:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827852.021532:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827852.021559:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827852/real 1787827852] req@ffff907d91149a40 x1874670011622016/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827852.021580:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011622016/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827852.021593:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827852.021598:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827852.021604:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011622016:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827852.021653:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011622144:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827852.021685:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827852.021703:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827852/real 1787827852] req@ffff907d9114ad00 x1874670011622144/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827852.021719:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011622144/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827852.021731:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827852.021736:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827852.021741:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011622144:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827852.148706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827852.148831:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827853.172681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827853.172882:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827854.196710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827854.196822:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827855.220742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827855.220823:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827856.244704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827856.244765:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787827857.140746:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827857.140753:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787827857.140757:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827857.140765:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3260 00000100:00080000:3.0:1787827857.140774:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827857.140780:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787827857.140826:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011622272/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827857.140848:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827857.140852:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787827857.140854:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827857.140859:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3260 00000100:00080000:3.0:1787827857.140862:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827857.140867:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827857.140876:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827857.140887:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011622272:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827857.140889:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011622400/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827857.140899:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827857.140902:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787827857.140904:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827857.140908:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3260 00000100:00080000:3.0:1787827857.140911:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827857.140914:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787827857.140919:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827857.140927:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011622528/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787827857.140934:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787827857.140937:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787827857.140939:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787827857.140942:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3260 00000100:00080000:3.0:1787827857.140945:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787827857.140948:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:0.0:1787827857.140949:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:3.0:1787827857.140953:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787827857.140962:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011622656/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827857.140994:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827857/real 1787827857] req@ffff907da562da40 x1874670011622272/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827857.141018:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011622272/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827857.141026:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827857.141029:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827857.141032:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011622272:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827857.141050:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011622400:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827857.141069:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827857.141078:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827857/real 1787827857] req@ffff907da562e940 x1874670011622400/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827857.141086:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011622400/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827857.141092:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827857.141095:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827857.141099:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011622400:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827857.141105:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011622528:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827857.141126:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827857.141137:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827857/real 1787827857] req@ffff907da562fc00 x1874670011622528/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827857.141149:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011622528/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827857.141162:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827857.141165:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827857.141167:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011622528:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827857.141171:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011622656:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827857.141205:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827857.141218:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827857/real 1787827857] req@ffff907da562f0c0 x1874670011622656/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827857.141231:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011622656/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827857.141240:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827857.141244:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827857.141247:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011622656:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827857.268684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827857.268768:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827858.292706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827858.292806:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827859.316775:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827859.316872:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827860.340701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827860.340821:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827861.364771:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827861.364840:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827862.260695:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827862.260704:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827862.260707:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827862.260715:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3266 00000100:00080000:2.0:1787827862.260724:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827862.260730:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827862.260762:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011622784/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827862.260782:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827862.260785:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827862.260786:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827862.260789:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3266 00000100:00080000:2.0:1787827862.260791:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827862.260793:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827862.260803:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827862.260812:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011622912/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827862.260819:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827862.260821:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827862.260823:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827862.260825:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3266 00000100:00080000:2.0:1787827862.260826:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827862.260831:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827862.260834:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011622784:192.168.203.104@tcp:250:kworker.0 00000008:00100000:2.0:1787827862.260835:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827862.260844:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011623040/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827862.260851:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827862.260852:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827862.260854:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827862.260856:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3266 00000100:00080000:2.0:1787827862.260858:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827862.260860:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827862.260863:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827862.260872:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011623168/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827862.260908:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827862.260946:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827862/real 1787827862] req@ffff907d91148b40 x1874670011622784/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827862.260960:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011622784/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827862.260967:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827862.260970:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827862.260973:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011622784:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827862.260979:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011622912:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827862.260997:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827862.261008:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827862/real 1787827862] req@ffff907d911492c0 x1874670011622912/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827862.261021:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011622912/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827862.261029:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827862.261033:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827862.261037:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011622912:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827862.261060:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011623040:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827862.261077:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827862.261086:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827862/real 1787827862] req@ffff907d91148f00 x1874670011623040/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827862.261093:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011623040/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827862.261099:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827862.261101:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827862.261103:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011623040:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827862.261107:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011623168:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827862.261117:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827862.261123:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827862/real 1787827862] req@ffff907d9114a940 x1874670011623168/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827862.261130:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011623168/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827862.261135:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827862.261137:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827862.261139:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011623168:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827862.388679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827862.388764:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827863.412773:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827863.412931:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827864.436748:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827864.436861:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827865.460701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827865.460760:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827866.484711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827866.484858:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827867.380723:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827867.380732:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827867.380735:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827867.380744:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3271 00000100:00080000:2.0:1787827867.380754:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827867.380761:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827867.380820:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011623296/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827867.380844:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827867.380847:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827867.380848:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827867.380851:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3271 00000100:00080000:2.0:1787827867.380854:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827867.380856:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827867.380864:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827867.380874:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011623424/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827867.380881:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827867.380884:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827867.380885:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827867.380888:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3271 00000100:00080000:2.0:1787827867.380890:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827867.380893:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827867.380897:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827867.380900:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011623296:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827867.380907:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011623552/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827867.380915:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827867.380918:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827867.380919:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827867.380921:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3271 00000100:00080000:2.0:1787827867.380923:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827867.380925:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827867.380928:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827867.380937:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011623680/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827867.380965:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827867.381005:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827867/real 1787827867] req@ffff907d9114a940 x1874670011623296/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827867.381021:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011623296/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827867.381028:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827867.381031:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827867.381034:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011623296:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827867.381039:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011623424:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827867.381054:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827867.381062:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827867/real 1787827867] req@ffff907d91148f00 x1874670011623424/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827867.381070:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011623424/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827867.381079:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827867.381082:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827867.381084:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011623424:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827867.381101:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011623552:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827867.381113:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827867.381119:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827867/real 1787827867] req@ffff907d911492c0 x1874670011623552/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827867.381126:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011623552/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827867.381132:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827867.381134:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827867.381136:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011623552:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827867.381139:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011623680:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827867.381148:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827867.381154:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827867/real 1787827867] req@ffff907d91148b40 x1874670011623680/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827867.381161:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011623680/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827867.381166:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827867.381167:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827867.381170:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011623680:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827867.508795:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827867.508860:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827868.532674:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827868.532766:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827869.556787:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827869.556891:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827870.580807:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827870.580973:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827871.604720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827871.604848:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827872.500851:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827872.500861:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827872.500866:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827872.500875:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3276 00000100:00080000:1.0:1787827872.500886:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827872.500893:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827872.500924:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011623808/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827872.500952:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827872.500954:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827872.500956:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827872.500959:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3276 00000100:00080000:1.0:1787827872.500973:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827872.500977:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827872.500986:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827872.500998:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011623808:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827872.501000:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011623936/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827872.501012:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827872.501014:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827872.501017:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827872.501023:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3276 00000100:00080000:1.0:1787827872.501026:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827872.501030:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827872.501036:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827872.501048:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011624064/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827872.501056:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827872.501060:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827872.501062:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827872.501068:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3276 00000800:00000100:2.0:1787827872.501071:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787827872.501073:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827872.501077:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827872.501081:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827872.501090:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011624192/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827872.501112:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827872/real 1787827872] req@ffff907d908a21c0 x1874670011623808/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827872.501134:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011623808/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827872.501147:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827872.501151:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827872.501154:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011623808:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827872.501291:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011623936:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827872.501321:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827872.501336:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827872/real 1787827872] req@ffff907d908a2580 x1874670011623936/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827872.501349:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011623936/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827872.501391:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827872.501395:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827872.501415:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011623936:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827872.501420:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011624064:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827872.501437:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827872.501445:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827872/real 1787827872] req@ffff907d908a2940 x1874670011624064/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827872.501462:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011624064/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827872.501488:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827872.501492:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827872.501495:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011624064:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827872.501503:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011624192:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827872.501525:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827872.501537:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827872/real 1787827872] req@ffff907d908a0000 x1874670011624192/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827872.501558:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011624192/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827872.501567:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827872.501571:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827872.501575:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011624192:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827872.628741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827872.628860:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827873.652748:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827873.652827:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827874.676858:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827874.676859:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827875.700747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827875.700922:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827876.724742:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827876.724748:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:1.0:1787827877.620774:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827877.620785:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827877.620789:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827877.620800:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3281 00000100:00080000:1.0:1787827877.620811:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827877.620820:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827877.620875:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011624320/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827877.620903:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827877.620909:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827877.620911:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827877.620914:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3281 00000100:00080000:1.0:1787827877.620917:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827877.620920:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827877.620929:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827877.620941:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011624448/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827877.620950:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827877.620953:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827877.620955:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827877.620975:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3281 00000100:00080000:1.0:1787827877.620978:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827877.620982:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827877.620995:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011624320:192.168.203.104@tcp:250:kworker.0 00000008:00100000:1.0:1787827877.620996:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827877.621046:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011624576/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827877.621075:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827877.621079:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827877.621081:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827877.621084:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3281 00000100:00080000:1.0:1787827877.621089:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827877.621093:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827877.621099:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787827877.621103:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827877.621119:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011624704/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827877.621190:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827877/real 1787827877] req@ffff907d908a1e00 x1874670011624320/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827877.621210:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011624320/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827877.621231:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827877.621234:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827877.621245:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011624320:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827877.621254:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011624448:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827877.621276:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827877.621286:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827877/real 1787827877] req@ffff907d908a2d00 x1874670011624448/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827877.621294:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011624448/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827877.621300:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827877.621303:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827877.621305:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011624448:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827877.621348:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011624576:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827877.621366:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827877.621375:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827877/real 1787827877] req@ffff907d908a0000 x1874670011624576/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827877.621389:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011624576/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827877.621398:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827877.621402:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827877.621406:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011624576:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827877.621414:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011624704:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827877.621434:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827877.621444:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827877/real 1787827877] req@ffff907d908a2940 x1874670011624704/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827877.621452:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011624704/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827877.621459:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827877.621461:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827877.621463:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011624704:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827877.748760:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827877.748849:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827878.772809:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827878.772955:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827879.796743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827879.796824:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827880.820691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827880.820773:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827881.844666:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827881.844733:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827882.740727:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827882.740736:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827882.740740:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827882.740748:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3286 00000100:00080000:2.0:1787827882.740757:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827882.740763:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827882.740795:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011624832/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827882.740822:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827882.740826:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827882.740828:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827882.740833:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3286 00000100:00080000:2.0:1787827882.740837:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827882.740843:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827882.740853:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827882.740864:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011624832:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827882.740870:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011624960/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827882.740880:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827882.740883:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827882.740884:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827882.740887:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3286 00000100:00080000:2.0:1787827882.740890:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827882.740893:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827882.740897:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827882.740906:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011625088/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827882.740913:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827882.740915:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827882.740916:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827882.740918:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3286 00000100:00080000:2.0:1787827882.740921:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787827882.740926:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787827882.740927:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827882.740934:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827882.740941:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011625216/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827882.740974:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827882/real 1787827882] req@ffff907d9114ad00 x1874670011624832/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827882.740989:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011624832/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827882.740997:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827882.741000:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827882.741003:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011624832:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827882.741021:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011624960:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827882.741039:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827882.741048:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827882/real 1787827882] req@ffff907d91149a40 x1874670011624960/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827882.741056:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011624960/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827882.741061:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827882.741063:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827882.741066:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011624960:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827882.741070:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011625088:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827882.741082:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827882.741089:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827882/real 1787827882] req@ffff907d9114a580 x1874670011625088/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827882.741097:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011625088/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827882.741102:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827882.741104:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827882.741106:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011625088:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827882.741109:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011625216:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827882.741118:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827882.741124:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827882/real 1787827882] req@ffff907d9114b840 x1874670011625216/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827882.741131:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011625216/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827882.741136:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827882.741138:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827882.741139:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011625216:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827882.868707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827882.868765:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827883.892698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827883.892756:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827884.916704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827884.916801:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827885.940716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827885.940799:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827886.964701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827886.964752:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827887.860713:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827887.860724:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827887.860732:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827887.860743:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3291 00000100:00080000:2.0:1787827887.860756:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827887.860763:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827887.860804:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011625344/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827887.860830:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827887.860834:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827887.860835:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827887.860838:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3291 00000100:00080000:2.0:1787827887.860841:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827887.860843:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827887.860853:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827887.860863:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011625472/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827887.860871:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827887.860875:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827887.860877:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827887.860880:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3291 00000100:00080000:2.0:1787827887.860883:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827887.860886:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827887.860890:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827887.860896:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011625600/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827887.860906:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787827887.860910:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011625344:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827887.860912:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827887.860920:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827887.860928:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3291 00000100:00080000:2.0:1787827887.860932:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827887.860937:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827887.860942:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827887.860959:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011625728/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827887.860982:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827887.861033:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827887/real 1787827887] req@ffff907d9114bc00 x1874670011625344/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827887.861054:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011625344/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827887.861067:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827887.861072:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827887.861077:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011625344:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827887.861086:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011625472:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827887.861107:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827887.861116:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827887/real 1787827887] req@ffff907d911483c0 x1874670011625472/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827887.861125:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011625472/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827887.861131:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827887.861134:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827887.861136:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011625472:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827887.861161:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011625600:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827887.861175:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827887.861182:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827887/real 1787827887] req@ffff907d9114b480 x1874670011625600/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827887.861189:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011625600/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827887.861195:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827887.861202:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827887.861205:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011625600:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827887.861210:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011625728:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827887.861221:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827887.861227:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827887/real 1787827887] req@ffff907d91149680 x1874670011625728/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827887.861235:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011625728/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827887.861240:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827887.861242:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827887.861244:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011625728:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827887.988693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827887.988794:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827889.012720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827889.012774:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827890.036700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827890.036783:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827891.060698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827891.060752:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827892.084678:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827892.084777:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827892.980720:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827892.980730:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827892.980735:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827892.980746:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3296 00000100:00080000:0.0:1787827892.980757:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827892.980763:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827892.980804:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011625856/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827892.980829:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827892.980833:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827892.980834:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827892.980837:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3296 00000100:00080000:0.0:1787827892.980840:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827892.980844:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827892.980855:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827892.980865:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011625984/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827892.980872:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011625856:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787827892.980875:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827892.980881:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787827892.980885:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827892.980889:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3296 00000100:00080000:0.0:1787827892.980891:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827892.980893:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827892.980896:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827892.980905:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011626112/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827892.980911:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827892.980914:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827892.980915:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827892.980918:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3296 00000100:00080000:0.0:1787827892.980920:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827892.980922:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827892.980925:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827892.980932:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011626240/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827892.980971:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827892.981023:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827892/real 1787827892] req@ffff907d90844780 x1874670011625856/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827892.981046:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011625856/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827892.981056:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827892.981060:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827892.981063:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011625856:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827892.981091:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011625984:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827892.981119:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827892.981133:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827892/real 1787827892] req@ffff907d90845680 x1874670011625984/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827892.981142:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011625984/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827892.981148:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827892.981151:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827892.981153:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011625984:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827892.981158:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011626112:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827892.981173:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827892.981180:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827892/real 1787827892] req@ffff907d90846d00 x1874670011626112/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827892.981188:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011626112/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827892.981193:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827892.981195:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827892.981198:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011626112:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827892.981202:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011626240:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827892.981228:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827892.981241:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827892/real 1787827892] req@ffff907d90846940 x1874670011626240/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827892.981255:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011626240/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827892.981264:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827892.981268:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827892.981270:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011626240:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827893.108691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827893.108794:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827894.132691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827894.132763:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827895.156677:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827895.156732:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827896.180683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827896.180752:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827897.204661:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827897.204750:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827898.100694:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827898.100706:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827898.100711:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827898.100723:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3301 00000100:00080000:2.0:1787827898.100736:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827898.100746:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827898.100792:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011626368/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827898.100815:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827898.100818:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827898.100819:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827898.100823:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3301 00000100:00080000:2.0:1787827898.100827:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827898.100830:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827898.100838:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827898.100850:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011626496/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827898.100852:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011626368:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827898.100866:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827898.100868:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827898.100870:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827898.100873:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3301 00000100:00080000:2.0:1787827898.100876:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827898.100878:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827898.100881:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827898.100890:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011626624/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827898.100896:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827898.100898:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827898.100900:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827898.100902:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3301 00000100:00080000:2.0:1787827898.100905:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827898.100907:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827898.100910:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827898.100917:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011626752/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827898.100935:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827898.100986:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827898/real 1787827898] req@ffff907d91149680 x1874670011626368/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827898.101002:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011626368/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827898.101010:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827898.101013:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827898.101016:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011626368:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827898.101033:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011626496:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827898.101052:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827898.101060:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827898/real 1787827898] req@ffff907d9114b480 x1874670011626496/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827898.101068:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011626496/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827898.101074:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827898.101075:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827898.101077:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011626496:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827898.101081:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011626624:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827898.101092:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827898.101098:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827898/real 1787827898] req@ffff907d911483c0 x1874670011626624/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827898.101110:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011626624/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827898.101115:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827898.101117:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827898.101119:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011626624:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827898.101122:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011626752:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827898.101133:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827898.101138:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827898/real 1787827898] req@ffff907d9114bc00 x1874670011626752/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827898.101145:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011626752/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827898.101151:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827898.101152:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827898.101154:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011626752:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827898.228683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827898.228762:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827899.252691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827899.252780:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827900.276691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827900.276751:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827901.300690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827901.300768:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827902.324737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827902.324824:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827903.220716:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827903.220725:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827903.220729:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827903.220739:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3306 00000100:00080000:0.0:1787827903.220750:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827903.220757:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827903.220793:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011626880/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827903.220821:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827903.220825:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827903.220827:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827903.220831:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3306 00000100:00080000:0.0:1787827903.220835:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827903.220840:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827903.220850:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827903.220853:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011626880:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827903.220867:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011627008/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827903.220880:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827903.220884:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787827903.220886:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827903.220891:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3306 00000100:00080000:0.0:1787827903.220895:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827903.220899:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827903.220904:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787827903.220919:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827903.220920:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011627136/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827903.220940:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827903.220944:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827903.220946:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827903.220950:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3306 00000100:00080000:0.0:1787827903.220956:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787827903.220960:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827903/real 1787827903] req@ffff907d90847480 x1874670011626880/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827903.220964:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827903.220973:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827903.220985:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011626880/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827903.220997:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011627264/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827903.221007:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827903.221015:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827903.221021:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011626880:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827903.221047:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011627008:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827903.221074:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827903.221087:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827903/real 1787827903] req@ffff907d90845a40 x1874670011627008/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827903.221096:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011627008/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827903.221102:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827903.221104:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827903.221107:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011627008:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827903.221111:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011627136:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827903.221125:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827903.221132:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827903/real 1787827903] req@ffff907d90844f00 x1874670011627136/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827903.221139:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011627136/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827903.221162:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827903.221165:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827903.221167:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011627136:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827903.221172:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011627264:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827903.221186:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827903.221194:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827903/real 1787827903] req@ffff907d90846940 x1874670011627264/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827903.221205:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011627264/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827903.221214:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827903.221217:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827903.221221:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011627264:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827903.348676:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827903.348733:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827904.372732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827904.372833:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827905.396682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827905.396759:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827906.420714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827906.420761:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827907.444692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827907.444800:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827908.340679:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827908.340688:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827908.340691:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827908.340700:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3312 00000100:00080000:0.0:1787827908.340710:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827908.340717:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827908.340748:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011627392/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827908.340769:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827908.340772:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827908.340773:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827908.340776:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3312 00000100:00080000:0.0:1787827908.340779:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827908.340781:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827908.340789:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827908.340800:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011627520/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827908.340809:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787827908.340810:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011627392:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787827908.340819:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787827908.340822:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827908.340826:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3312 00000100:00080000:0.0:1787827908.340829:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827908.340832:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827908.340836:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827908.340844:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011627648/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827908.340851:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827908.340854:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827908.340856:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827908.340859:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3312 00000100:00080000:0.0:1787827908.340862:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827908.340865:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827908.340868:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827908.340877:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011627776/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787827908.340885:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827908.340925:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827908/real 1787827908] req@ffff907d90846d00 x1874670011627392/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827908.340940:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011627392/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827908.340948:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827908.340951:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827908.340955:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011627392:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827908.340973:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011627520:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827908.340991:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827908.341000:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827908/real 1787827908] req@ffff907d90845680 x1874670011627520/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827908.341009:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011627520/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827908.341015:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827908.341017:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827908.341021:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011627520:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827908.341025:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011627648:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827908.341039:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827908.341046:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827908/real 1787827908] req@ffff907d90844780 x1874670011627648/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827908.341054:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011627648/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827908.341060:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827908.341062:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827908.341064:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011627648:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827908.341068:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011627776:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827908.341078:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827908.341084:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827908/real 1787827908] req@ffff907d908452c0 x1874670011627776/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827908.341091:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011627776/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827908.341097:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827908.341101:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827908.341105:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011627776:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827908.468672:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827908.468755:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827909.492667:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827909.492787:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827910.516702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827910.516782:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827911.540679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827911.540734:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827912.564680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827912.564736:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827913.460725:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827913.460742:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827913.460747:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827913.460762:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3317 00000100:00080000:2.0:1787827913.460777:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827913.460786:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827913.460831:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011627904/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827913.460869:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827913.460874:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827913.460876:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827913.460881:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3317 00000100:00080000:2.0:1787827913.460886:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827913.460891:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827913.460903:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827913.460911:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011627904:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827913.460921:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011628032/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827913.460933:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827913.460937:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827913.460939:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827913.460943:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3317 00000100:00080000:2.0:1787827913.460947:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827913.460952:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827913.460957:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827913.460968:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011628160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827913.460979:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827913.460982:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827913.460985:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827913.460989:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3317 00000100:00080000:2.0:1787827913.460993:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827913.460997:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827913.461003:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827913.461017:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011628288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827913.461036:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011628032:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827913.461101:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011628160:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827913.461135:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011628288:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827913.588687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827913.588773:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827914.612706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827914.612801:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827915.636676:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827915.636740:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787827916.532805:0:2338:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:3.0:1787827916.532860:0:2338:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787827916.532875:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787827916.532917:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787827916.532927:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787827916.532938:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:0.0:1787827916.532967:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827913/real 1787827916] req@ffff907d9114a580 x1874670011628032/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827916.532994:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011628032/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827916.533008:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827916.533013:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827916.533018:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011628032:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827916.533027:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827913/real 1787827916] req@ffff907d91149a40 x1874670011628160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827916.533041:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011628160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827916.533062:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827916.533067:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827916.533071:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011628160:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827916.533078:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827913/real 1787827916] req@ffff907d9114ad00 x1874670011628288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827916.533091:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011628288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827916.533100:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827916.533104:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827916.533108:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011628288:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827916.533116:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827913/real 1787827916] req@ffff907d9114b840 x1874670011627904/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827916.533129:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011627904/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827916.533138:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827916.533141:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827916.533145:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011627904:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827916.660681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827916.660751:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827917.684708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827917.684773:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827918.580714:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827918.580722:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827918.580725:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827918.580734:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3322 00000100:00080000:0.0:1787827918.580743:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827918.580749:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827918.580782:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011628416/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827918.580810:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827918.580814:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827918.580816:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827918.580822:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3322 00000100:00080000:0.0:1787827918.580827:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827918.580833:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827918.580841:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011628416:192.168.203.104@tcp:250:kworker.0 00000008:00100000:0.0:1787827918.580843:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827918.580863:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011628544/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827918.580873:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827918.580876:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787827918.580877:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827918.580881:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3322 00000100:00080000:0.0:1787827918.580885:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827918.580888:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827918.580891:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827918.580901:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011628672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827918.580907:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827918.580910:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827918.580913:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787827918.580915:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787827918.580918:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3322 00000100:00080000:0.0:1787827918.580922:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827918.580924:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827918.580927:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827918.580935:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011628800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827918.580962:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827918/real 1787827918] req@ffff907d90844000 x1874670011628416/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827918.580982:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011628416/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827918.580993:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827918.580996:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827918.581000:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011628416:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827918.581024:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011628544:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827918.581063:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827918.581077:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827918/real 1787827918] req@ffff907d908443c0 x1874670011628544/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827918.581090:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011628544/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827918.581099:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827918.581102:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827918.581106:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011628544:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827918.581112:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011628672:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827918.581135:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827918.581147:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827918/real 1787827918] req@ffff907d90844b40 x1874670011628672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827918.581158:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011628672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827918.581166:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827918.581170:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827918.581173:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011628672:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827918.581179:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011628800:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827918.581198:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827918.581210:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827918/real 1787827918] req@ffff907d908470c0 x1874670011628800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827918.581221:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011628800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827918.581229:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827918.581239:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827918.581242:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011628800:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827918.708673:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827918.708751:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827919.732709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827919.732762:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827920.756676:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827920.756728:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827921.780760:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827921.780832:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827922.804717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827922.804819:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827923.700730:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827923.700742:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827923.700747:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827923.700757:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3327 00000100:00080000:1.0:1787827923.700779:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827923.700787:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827923.700847:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011628928/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827923.700876:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827923.700880:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827923.700882:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827923.700887:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3327 00000100:00080000:1.0:1787827923.700891:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827923.700896:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827923.700907:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827923.700915:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011628928:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827923.700925:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011629056/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827923.700939:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827923.700943:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827923.700945:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827923.700960:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3327 00000100:00080000:1.0:1787827923.700965:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827923.700971:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827923.700976:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787827923.700992:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827923.700994:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011629184/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827923.701033:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827923.701037:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827923.701039:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827923.701044:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3327 00000100:00080000:1.0:1787827923.701049:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827923.701054:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827923.701060:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827923.701077:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827923/real 1787827923] req@ffff907d908a2580 x1874670011628928/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787827923.701079:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011629312/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827923.701111:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011628928/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827923.701121:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827923.701125:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827923.701129:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011628928:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827923.701147:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011629056:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827923.701164:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827923.701173:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827923/real 1787827923] req@ffff907d908a21c0 x1874670011629056/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827923.701182:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011629056/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827923.701188:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827923.701190:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827923.701192:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011629056:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827923.701196:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011629184:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827923.701207:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827923.701213:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827923/real 1787827923] req@ffff907d908a0f00 x1874670011629184/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827923.701221:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011629184/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827923.701226:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827923.701228:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827923.701230:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011629184:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827923.701233:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011629312:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827923.701242:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827923.701248:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827923/real 1787827923] req@ffff907d908a12c0 x1874670011629312/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827923.701255:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011629312/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827923.701274:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827923.701277:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827923.701279:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011629312:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827923.828676:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827923.828726:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827924.852746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827924.852916:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827925.876784:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827925.876920:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827926.900728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827926.900931:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827927.924728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827927.924922:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827928.820785:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827928.820799:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827928.820805:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827928.820818:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3332 00000100:00080000:2.0:1787827928.820832:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827928.820841:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827928.820887:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011629440/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827928.820929:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827928.820934:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827928.820937:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827928.820942:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3332 00000100:00080000:2.0:1787827928.820946:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827928.820950:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827928.820962:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827928.821005:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011629568/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827928.821022:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827928.821027:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827928.821030:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827928.821035:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3332 00000100:00080000:2.0:1787827928.821042:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827928.821051:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827928.821059:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827928.821060:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011629440:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827928.821073:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011629696/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827928.821089:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827928.821094:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827928.821096:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827928.821100:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3332 00000100:00080000:2.0:1787827928.821104:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827928.821109:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827928.821115:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827928.821130:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011629824/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827928.821159:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827928.821226:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827928/real 1787827928] req@ffff907d91148b40 x1874670011629440/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827928.821261:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011629440/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827928.821276:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827928.821281:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827928.821287:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011629440:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827928.821297:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011629568:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827928.821324:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827928.821339:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827928/real 1787827928] req@ffff907d911492c0 x1874670011629568/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827928.821354:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011629568/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827928.821378:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827928.821381:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827928.821386:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011629568:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827928.821415:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011629696:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827928.821440:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827928.821453:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827928/real 1787827928] req@ffff907d91148f00 x1874670011629696/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827928.821468:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011629696/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827928.821486:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827928.821490:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827928.821494:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011629696:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827928.821501:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011629824:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827928.821518:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827928.821529:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827928/real 1787827928] req@ffff907d9114a940 x1874670011629824/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827928.821543:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011629824/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827928.821553:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827928.821556:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827928.821559:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011629824:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827928.948778:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827928.948872:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827929.972741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827929.972882:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827930.996705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827930.996810:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827932.020675:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827932.020750:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787827933.044693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827933.044864:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827933.940771:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827933.940783:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827933.940788:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827933.940798:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3337 00000100:00080000:0.0:1787827933.940811:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827933.940819:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827933.940860:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011629952/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827933.940893:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827933.940898:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827933.940900:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827933.940904:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3337 00000100:00080000:0.0:1787827933.940908:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827933.940912:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827933.940923:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827933.940959:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011630080/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827933.940969:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011629952:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787827933.940977:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827933.940986:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787827933.940989:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827933.940997:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3337 00000100:00080000:0.0:1787827933.941003:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827933.941009:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827933.941018:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827933.941039:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011630208/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827933.941057:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827933.941063:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827933.941066:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827933.941077:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3337 00000800:00000100:2.0:1787827933.941077:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787827933.941088:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827933.941093:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827933.941101:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827933.941119:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011630336/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827933.941142:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827933/real 1787827933] req@ffff907d90847c00 x1874670011629952/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827933.941166:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011629952/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827933.941178:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827933.941182:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827933.941186:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011629952:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827933.941211:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011630080:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827933.941234:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827933.941247:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827933/real 1787827933] req@ffff907d90846580 x1874670011630080/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827933.941260:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011630080/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827933.941268:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827933.941282:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827933.941286:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011630080:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827933.941292:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011630208:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827933.941310:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827933.941321:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827933/real 1787827933] req@ffff907d90845e00 x1874670011630208/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827933.941339:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011630208/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827933.941348:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827933.941350:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827933.941353:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011630208:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827933.941359:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011630336:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827933.941374:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827933.941383:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827933/real 1787827933] req@ffff907d908470c0 x1874670011630336/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827933.941404:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011630336/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827933.941412:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827933.941415:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827933.941418:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011630336:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827934.068762:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827934.068892:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827935.092796:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827935.092895:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827936.116698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827936.116746:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827937.140673:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827937.140744:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827938.164726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827938.164794:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827939.060710:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827939.060722:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827939.060727:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827939.060739:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3342 00000100:00080000:0.0:1787827939.060753:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827939.060763:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827939.060824:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011630464/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827939.060884:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827939.060891:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827939.060913:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827939.060922:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3342 00000100:00080000:0.0:1787827939.060929:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827939.060936:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827939.060963:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827939.060991:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011630592/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827939.061015:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827939.061022:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1787827939.061025:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011630464:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787827939.061026:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827939.061046:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3342 00000100:00080000:0.0:1787827939.061054:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827939.061062:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827939.061073:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827939.061099:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011630720/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827939.061119:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827939.061125:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827939.061128:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787827939.061138:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787827939.061144:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3342 00000100:00080000:0.0:1787827939.061152:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827939.061159:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827939.061167:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827939.061185:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011630848/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787827939.061200:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827939/real 1787827939] req@ffff907d90844b40 x1874670011630464/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827939.061228:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011630464/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827939.061242:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827939.061247:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827939.061252:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011630464:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827939.061279:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011630592:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787827939.061306:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827939.061320:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827939/real 1787827939] req@ffff907d908443c0 x1874670011630592/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827939.061334:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011630592/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827939.061345:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827939.061351:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827939.061357:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011630592:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787827939.061368:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011630720:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827939.061403:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827939.061425:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827939/real 1787827939] req@ffff907d90844000 x1874670011630720/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827939.061449:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011630720/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827939.061477:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827939.061483:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827939.061489:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011630720:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787827939.061501:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011630848:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787827939.061539:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787827939.061558:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827939/real 1787827939] req@ffff907d908452c0 x1874670011630848/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827939.061572:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011630848/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827939.061582:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787827939.061586:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787827939.061589:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011630848:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827939.188731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827939.188787:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827940.212695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827940.212819:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827941.236731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827941.236877:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827942.260752:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827942.260826:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827943.284699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827943.284727:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827944.180749:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827944.180756:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827944.180760:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827944.180769:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3347 00000100:00080000:2.0:1787827944.180793:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827944.180803:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827944.180847:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011630976/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827944.180910:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827944.180914:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827944.180916:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827944.180921:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3347 00000100:00080000:2.0:1787827944.180925:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827944.180930:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827944.180941:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827944.180958:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011630976:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827944.180960:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011631104/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827944.180977:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827944.180980:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827944.180982:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827944.180986:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3347 00000100:00080000:2.0:1787827944.180991:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827944.181009:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827944.181016:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827944.181030:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011631232/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827944.181034:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787827944.181045:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827944.181050:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827944.181052:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827944.181057:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3347 00000100:00080000:2.0:1787827944.181062:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827944.181066:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827944.181071:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827944.181075:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827944/real 1787827944] req@ffff907d9114a940 x1874670011630976/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787827944.181085:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011631360/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827944.181119:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011630976/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827944.181140:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827944.181146:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827944.181152:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011630976:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827944.181176:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011631104:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827944.181203:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827944.181218:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827944/real 1787827944] req@ffff907d91148f00 x1874670011631104/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827944.181232:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011631104/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827944.181242:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827944.181246:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827944.181266:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011631104:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827944.181273:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011631232:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827944.181293:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827944.181304:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827944/real 1787827944] req@ffff907d911492c0 x1874670011631232/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827944.181344:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011631232/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827944.181355:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827944.181358:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827944.181362:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011631232:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827944.181369:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011631360:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827944.181390:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827944.181403:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827944/real 1787827944] req@ffff907d91148b40 x1874670011631360/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827944.181416:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011631360/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827944.181426:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827944.181430:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827944.181434:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011631360:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827944.308698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827944.308754:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827945.332703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827945.332759:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827946.356723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827946.356809:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787827947.380863:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827947.380925:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787827948.404713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827948.404876:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827949.300678:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827949.300687:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827949.300690:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827949.300699:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3353 00000100:00080000:2.0:1787827949.300708:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827949.300714:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827949.300746:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011631488/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827949.300769:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827949.300771:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827949.300773:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827949.300776:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3353 00000100:00080000:2.0:1787827949.300778:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827949.300780:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827949.300788:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827949.300797:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011631616/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827949.300806:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787827949.300809:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011631488:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827949.300810:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827949.300813:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827949.300821:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3353 00000100:00080000:2.0:1787827949.300838:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827949.300842:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827949.300846:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827949.300856:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011631744/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827949.300862:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827949.300865:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827949.300867:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827949.300869:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3353 00000100:00080000:2.0:1787827949.300872:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827949.300874:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827949.300877:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827949.300885:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011631872/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827949.300901:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827949.300952:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827949/real 1787827949] req@ffff907d9114b840 x1874670011631488/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827949.300973:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011631488/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827949.300984:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827949.300987:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827949.300990:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011631488:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827949.301009:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011631616:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827949.301029:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827949.301039:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827949/real 1787827949] req@ffff907d9114ad00 x1874670011631616/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827949.301047:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011631616/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827949.301053:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827949.301055:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827949.301058:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011631616:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827949.301061:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011631744:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827949.301073:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827949.301082:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827949/real 1787827949] req@ffff907d91149a40 x1874670011631744/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827949.301095:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011631744/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827949.301104:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827949.301107:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827949.301111:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011631744:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827949.301117:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011631872:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827949.301138:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827949.301147:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827949/real 1787827949] req@ffff907d9114a580 x1874670011631872/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827949.301154:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011631872/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827949.301160:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827949.301163:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827949.301165:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011631872:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827949.428747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827949.428874:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827950.452688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827950.452765:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827951.476734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827951.476918:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827952.500771:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827952.500916:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827953.524760:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827953.524872:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827954.420722:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827954.420733:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827954.420740:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827954.420753:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3358 00000100:00080000:2.0:1787827954.420768:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827954.420781:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827954.420868:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011632000/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827954.420905:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827954.420909:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827954.420911:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827954.420916:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3358 00000100:00080000:2.0:1787827954.420920:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827954.420924:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827954.420935:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827954.420969:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011632128/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827954.420984:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787827954.420987:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011632000:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827954.420990:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827954.420996:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827954.421006:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3358 00000100:00080000:2.0:1787827954.421010:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827954.421015:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827954.421021:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827954.421036:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011632256/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827954.421070:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827954.421073:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827954.421075:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827954.421078:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3358 00000100:00080000:2.0:1787827954.421082:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827954.421084:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827954.421088:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827954.421101:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011632384/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827954.421103:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827954.421178:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827954/real 1787827954] req@ffff907d9114bc00 x1874670011632000/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827954.421202:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011632000/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827954.421215:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827954.421220:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827954.421225:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011632000:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827954.421251:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011632128:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827954.421290:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827954.421304:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827954/real 1787827954] req@ffff907d911483c0 x1874670011632128/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827954.421321:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011632128/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827954.421354:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827954.421375:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827954.421383:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011632128:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827954.421396:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011632256:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827954.421431:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827954.421447:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827954/real 1787827954] req@ffff907d9114b480 x1874670011632256/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827954.421461:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011632256/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827954.421471:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827954.421476:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827954.421480:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011632256:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827954.421487:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011632384:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827954.421506:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827954.421516:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827954/real 1787827954] req@ffff907d91149680 x1874670011632384/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827954.421535:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011632384/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827954.421551:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827954.421572:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827954.421576:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011632384:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827954.548737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827954.548892:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827955.572752:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827955.572811:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827956.596696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827956.596866:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827957.620764:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827957.620810:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827958.644693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827958.644756:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827959.540787:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827959.540799:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827959.540814:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827959.540831:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3363 00000100:00080000:1.0:1787827959.540861:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827959.540870:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827959.540909:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011632512/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827959.540942:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827959.540946:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827959.540948:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827959.540952:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3363 00000100:00080000:1.0:1787827959.540956:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827959.540961:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827959.540971:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827959.540986:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011632640/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827959.540999:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827959.541005:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827959.541009:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787827959.541011:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011632512:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827959.541017:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3363 00000100:00080000:1.0:1787827959.541028:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827959.541033:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827959.541038:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827959.541078:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011632768/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827959.541092:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827959.541097:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827959.541100:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827959.541104:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3363 00000100:00080000:1.0:1787827959.541122:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827959.541127:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827959.541135:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787827959.541143:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787827959.541156:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011632896/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827959.541219:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827959/real 1787827959] req@ffff907d908a1680 x1874670011632512/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827959.541241:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011632512/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827959.541252:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827959.541256:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827959.541260:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011632512:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827959.541306:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011632640:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827959.541328:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827959.541338:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827959/real 1787827959] req@ffff907d908a3840 x1874670011632640/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827959.541348:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011632640/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827959.541365:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827959.541368:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827959.541371:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011632640:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827959.541376:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011632768:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827959.541390:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827959.541397:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827959/real 1787827959] req@ffff907d908a12c0 x1874670011632768/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827959.541411:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011632768/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827959.541417:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827959.541419:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827959.541422:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011632768:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827959.541427:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011632896:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827959.541439:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827959.541445:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827959/real 1787827959] req@ffff907d908a0f00 x1874670011632896/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827959.541454:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011632896/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827959.541460:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827959.541462:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827959.541464:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011632896:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827959.668703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827959.668803:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827960.692848:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827960.692876:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827961.716755:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827961.716857:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827962.740823:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827962.740919:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827963.764801:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827963.764947:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827964.660765:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827964.660779:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827964.660786:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827964.660799:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3368 00000100:00080000:1.0:1787827964.660814:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827964.660823:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827964.660907:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011633024/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827964.660982:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827964.660987:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827964.660989:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827964.660995:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3368 00000100:00080000:1.0:1787827964.660999:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827964.661004:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827964.661017:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827964.661034:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011633152/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827964.661047:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827964.661054:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787827964.661056:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011633024:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827964.661057:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827964.661068:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3368 00000100:00080000:1.0:1787827964.661074:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827964.661079:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827964.661086:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827964.661102:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011633280/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827964.661114:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827964.661118:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827964.661121:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827964.661126:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3368 00000100:00080000:1.0:1787827964.661131:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827964.661135:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:0.0:1787827964.661139:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:1.0:1787827964.661144:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827964.661160:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011633408/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787827964.661203:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827964/real 1787827964] req@ffff907d908a21c0 x1874670011633024/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827964.661220:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011633024/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827964.661253:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827964.661259:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827964.661265:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011633024:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827964.661313:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011633152:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827964.661347:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827964.661388:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827964/real 1787827964] req@ffff907d908a2580 x1874670011633152/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827964.661397:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011633152/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827964.661403:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827964.661406:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827964.661408:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011633152:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827964.661412:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011633280:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827964.661425:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827964.661448:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827964/real 1787827964] req@ffff907d908a2940 x1874670011633280/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827964.661463:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011633280/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827964.661474:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827964.661477:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827964.661481:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011633280:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827964.661493:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011633408:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827964.661514:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827964.661525:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827964/real 1787827964] req@ffff907d908a0000 x1874670011633408/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827964.661539:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011633408/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827964.661549:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827964.661552:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827964.661556:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011633408:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827964.788791:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787827964.788855:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827965.812786:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827965.812887:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827966.836782:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827966.836904:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827967.860790:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827967.860870:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827968.884786:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827968.884917:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827969.780784:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827969.780801:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827969.780808:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827969.780824:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3373 00000100:00080000:2.0:1787827969.780843:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827969.780855:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827969.780948:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011633536/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827969.781008:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827969.781017:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827969.781020:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827969.781027:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3373 00000100:00080000:2.0:1787827969.781033:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827969.781041:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827969.781057:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827969.781082:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011633664/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827969.781111:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827969.781116:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827969.781125:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787827969.781130:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011633536:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787827969.781136:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3373 00000100:00080000:2.0:1787827969.781152:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827969.781162:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827969.781169:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827969.781187:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011633792/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827969.781200:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827969.781219:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827969.781224:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827969.781230:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3373 00000100:00080000:2.0:1787827969.781235:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827969.781240:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827969.781247:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827969.781271:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011633920/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827969.781288:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827969.781418:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827969/real 1787827969] req@ffff907d91149680 x1874670011633536/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827969.781459:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011633536/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827969.781482:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827969.781491:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827969.781500:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011633536:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827969.781543:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011633664:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827969.781604:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827969.781704:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827969/real 1787827969] req@ffff907d9114b480 x1874670011633664/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827969.781734:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011633664/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827969.781752:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827969.781759:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827969.781767:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011633664:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827969.781780:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011633792:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827969.781819:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827969.781856:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827969/real 1787827969] req@ffff907d911483c0 x1874670011633792/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827969.781880:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011633792/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827969.781897:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827969.781904:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827969.781911:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011633792:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827969.781923:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011633920:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827969.781958:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827969.781978:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827969/real 1787827969] req@ffff907d9114bc00 x1874670011633920/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827969.782002:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011633920/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827969.782020:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827969.782026:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827969.782032:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011633920:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827969.908784:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827969.908877:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827970.932778:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827970.932874:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827971.956785:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827971.956877:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827972.980825:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827972.980975:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827974.004749:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827974.004789:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827974.900757:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827974.900768:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827974.900781:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827974.900797:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3378 00000100:00080000:2.0:1787827974.900810:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827974.900818:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827974.900865:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011634048/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827974.900904:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827974.900909:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827974.900911:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827974.900917:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3378 00000100:00080000:2.0:1787827974.900921:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827974.900926:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827974.900967:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787827974.900980:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011634176/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827974.900990:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827974.900993:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827974.900995:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827974.900999:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3378 00000100:00080000:2.0:1787827974.901002:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827974.901006:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827974.901010:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827974.901017:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011634048:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827974.901019:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011634304/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827974.901032:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827974.901035:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827974.901036:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827974.901038:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3378 00000100:00080000:2.0:1787827974.901040:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827974.901043:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827974.901047:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827974.901055:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011634432/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787827974.901114:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827974.901165:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827974/real 1787827974] req@ffff907d9114a580 x1874670011634048/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827974.901182:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011634048/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787827999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827974.901192:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827974.901196:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827974.901199:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011634048:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787827974.901205:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011634176:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787827974.901222:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827974.901230:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827974/real 1787827974] req@ffff907d91149a40 x1874670011634176/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827974.901238:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011634176/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787827999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827974.901244:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827974.901246:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827974.901248:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011634176:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787827974.901281:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011634304:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827974.901293:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827974.901301:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827974/real 1787827974] req@ffff907d9114ad00 x1874670011634304/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827974.901309:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011634304/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827974.901314:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827974.901316:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827974.901318:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011634304:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787827974.901322:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011634432:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787827974.901332:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787827974.901338:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827974/real 1787827974] req@ffff907d9114b840 x1874670011634432/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787827974.901345:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011634432/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787827999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787827974.901350:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787827974.901352:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787827974.901354:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011634432:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827975.028741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827975.028809:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827976.052781:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827976.052874:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827977.076723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827977.076802:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827978.100779:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827978.100905:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827979.124716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827979.124838:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827980.020811:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827980.020826:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827980.020842:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827980.020861:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3383 00000100:00080000:2.0:1787827980.020879:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827980.020891:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827980.020957:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011634560/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827980.021006:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827980.021012:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827980.021015:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827980.021028:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3383 00000100:00080000:2.0:1787827980.021037:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827980.021046:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827980.021067:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827980.021083:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011634560:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827980.021096:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011634688/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827980.021119:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827980.021126:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827980.021130:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827980.021141:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3383 00000100:00080000:2.0:1787827980.021148:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827980.021155:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827980.021167:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827980.021188:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011634816/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827980.021202:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827980.021207:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827980.021209:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827980.021217:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3383 00000100:00080000:2.0:1787827980.021297:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827980.021304:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827980.021310:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827980.021327:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011634944/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827980.021535:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011634688:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827980.021586:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011634816:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827980.021671:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011634944:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827980.148722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827980.148829:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827981.172764:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827981.172856:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827982.196723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827982.196820:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787827983.093049:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787827983.093149:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787827983.093184:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787827983.093279:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787827983.093301:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787827983.093323:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:3.0:1787827983.093357:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827980/real 1787827983] req@ffff907d911492c0 x1874670011634688/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827983.093411:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011634688/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827983.093438:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827983.093445:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827983.093452:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011634688:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827983.093463:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827980/real 1787827983] req@ffff907d91148f00 x1874670011634816/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827983.093481:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011634816/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827983.093493:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827983.093498:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827983.093503:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011634816:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827983.093511:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827980/real 1787827983] req@ffff907d9114a940 x1874670011634944/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827983.093524:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011634944/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827983.093533:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827983.093537:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827983.093541:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011634944:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827983.093548:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827980/real 1787827983] req@ffff907d91148b40 x1874670011634560/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827983.093562:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011634560/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827983.093571:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827983.093575:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827983.093579:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011634560:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787827983.220789:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827983.220837:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827984.244767:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827984.244932:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787827985.140784:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827985.140799:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787827985.140806:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827985.140823:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3388 00000100:00080000:0.0:1787827985.140840:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827985.140851:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787827985.140905:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011635072/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827985.140946:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827985.140951:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787827985.140954:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827985.140959:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3388 00000100:00080000:0.0:1787827985.140964:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827985.140968:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827985.140982:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787827985.141000:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011635200/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827985.141016:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827985.141024:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1787827985.141026:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011635072:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787827985.141028:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827985.141042:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3388 00000100:00080000:0.0:1787827985.141046:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827985.141052:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827985.141058:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827985.141074:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011635328/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787827985.141125:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787827985.141133:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787827985.141136:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787827985.141144:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3388 00000800:00000100:3.0:1787827985.141156:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787827985.141161:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787827985.141174:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787827985.141186:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787827985.141216:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011635456/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827985.141271:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827985/real 1787827985] req@ffff907d90844780 x1874670011635072/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827985.141355:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011635072/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827985.141377:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827985.141386:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827985.141395:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011635072:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827985.141441:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011635200:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827985.141500:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827985.141531:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827985/real 1787827985] req@ffff907d90845680 x1874670011635200/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827985.141562:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011635200/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827985.141652:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827985.141661:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827985.141670:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011635200:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827985.141685:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011635328:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827985.141735:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827985.141762:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827985/real 1787827985] req@ffff907d90846d00 x1874670011635328/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827985.141788:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011635328/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827985.142053:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827985.142062:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827985.142069:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011635328:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827985.142083:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011635456:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827985.142131:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827985.142155:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827985/real 1787827985] req@ffff907d90846940 x1874670011635456/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827985.142181:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011635456/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827985.142198:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827985.142205:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827985.142212:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011635456:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827985.268773:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827985.268888:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827986.292777:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827986.292880:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827987.316760:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827987.316844:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827988.340693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827988.340879:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827989.364728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827989.364840:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787827990.260736:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827990.260749:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787827990.260755:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827990.260767:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3394 00000100:00080000:2.0:1787827990.260782:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827990.260793:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787827990.260855:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011635584/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827990.260903:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827990.260911:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787827990.260914:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827990.260923:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3394 00000100:00080000:2.0:1787827990.260931:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827990.260941:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827990.260959:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787827990.260990:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011635584:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787827990.260991:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011635712/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827990.261027:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827990.261035:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787827990.261039:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827990.261048:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3394 00000100:00080000:2.0:1787827990.261083:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787827990.261094:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:3.0:1787827990.261108:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:2.0:1787827990.261113:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827990.261143:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011635840/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787827990.261158:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787827990.261163:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787827990.261166:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787827990.261173:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3394 00000100:00080000:2.0:1787827990.261180:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787827990.261183:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827990/real 1787827990] req@ffff907d91148b40 x1874670011635584/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787827990.261191:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787827990.261207:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787827990.261227:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011635968/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827990.261234:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011635584/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827990.261291:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827990.261298:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827990.261306:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011635584:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827990.261337:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011635712:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827990.261372:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827990.261390:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827990/real 1787827990] req@ffff907d9114a940 x1874670011635712/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827990.261405:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011635712/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827990.261416:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827990.261420:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827990.261424:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011635712:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827990.261431:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011635840:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827990.261452:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827990.261464:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827990/real 1787827990] req@ffff907d91148f00 x1874670011635840/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827990.261478:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011635840/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827990.261488:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827990.261491:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827990.261495:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011635840:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827990.261501:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011635968:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827990.261520:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827990.261531:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827990/real 1787827990] req@ffff907d911492c0 x1874670011635968/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827990.261590:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011635968/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827990.261602:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827990.261656:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827990.261664:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011635968:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827990.388750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827990.388833:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827991.412732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827991.412919:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827992.436820:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827992.436838:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827993.460782:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827993.460871:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827994.484701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827994.484895:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787827995.380771:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827995.380781:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787827995.380803:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827995.380816:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3399 00000100:00080000:1.0:1787827995.380831:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827995.380841:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787827995.380893:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011636096/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827995.380941:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827995.380949:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787827995.380953:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827995.380961:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3399 00000100:00080000:1.0:1787827995.380969:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827995.380994:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827995.381010:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787827995.381038:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011636224/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827995.381038:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011636096:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787827995.381061:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827995.381065:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787827995.381067:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827995.381072:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3399 00000100:00080000:1.0:1787827995.381077:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827995.381081:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827995.381088:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827995.381103:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011636352/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787827995.381114:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787827995.381117:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787827995.381119:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787827995.381125:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3399 00000800:00000100:3.0:1787827995.381132:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787827995.381135:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787827995.381142:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787827995.381148:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787827995.381173:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011636480/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787827995.381205:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827995/real 1787827995] req@ffff907d908a2d00 x1874670011636096/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827995.381240:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011636096/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827995.381259:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827995.381266:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827995.381272:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011636096:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787827995.381300:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011636224:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787827995.381329:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827995.381344:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827995/real 1787827995] req@ffff907d908a1e00 x1874670011636224/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827995.381382:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011636224/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827995.381407:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827995.381412:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827995.381418:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011636224:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787827995.381429:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011636352:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827995.381460:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827995.381473:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827995/real 1787827995] req@ffff907d908a0000 x1874670011636352/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827995.381485:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011636352/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827995.381494:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827995.381498:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827995.381501:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011636352:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787827995.381530:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011636480:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787827995.381578:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787827995.381601:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787827995/real 1787827995] req@ffff907d908a2940 x1874670011636480/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787827995.381678:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011636480/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787827995.381707:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787827995.381710:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787827995.381715:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011636480:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787827995.508744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787827995.508900:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827996.532774:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827996.532889:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827997.556834:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827997.556936:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827998.580731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787827998.580881:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787827999.604784:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787827999.604915:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828000.500698:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828000.500707:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828000.500711:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828000.500719:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3404 00000100:00080000:2.0:1787828000.500728:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828000.500734:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828000.500765:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011636608/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828000.500786:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828000.500789:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828000.500790:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828000.500793:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3404 00000100:00080000:2.0:1787828000.500796:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828000.500798:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828000.500806:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828000.500821:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011636736/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828000.500829:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828000.500832:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828000.500833:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828000.500836:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3404 00000100:00080000:2.0:1787828000.500838:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828000.500841:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828000.500844:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828000.500850:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011636608:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828000.500854:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011636864/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828000.500865:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828000.500867:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828000.500868:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828000.500871:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3404 00000100:00080000:2.0:1787828000.500875:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828000.500879:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828000.500884:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828000.500896:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011636992/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828000.500923:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828000.500972:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828000/real 1787828000] req@ffff907d9114b840 x1874670011636608/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828000.500990:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011636608/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828000.501013:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828000.501018:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828000.501022:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011636608:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828000.501029:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011636736:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828000.501052:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828000.501065:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828000/real 1787828000] req@ffff907d9114ad00 x1874670011636736/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828000.501078:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011636736/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828000.501086:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828000.501089:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828000.501092:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011636736:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828000.501124:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011636864:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828000.501148:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828000.501162:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828000/real 1787828000] req@ffff907d91149a40 x1874670011636864/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828000.501175:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011636864/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828000.501183:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828000.501186:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828000.501189:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011636864:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828000.501195:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011636992:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828000.501211:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828000.501221:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828000/real 1787828000] req@ffff907d9114a580 x1874670011636992/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828000.501233:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011636992/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828000.501241:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828000.501245:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828000.501248:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011636992:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828000.628711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828000.628739:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828001.652715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828001.652885:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828002.676797:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828002.676839:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828003.700735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828003.700810:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828004.724768:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828004.724891:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828005.620739:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828005.620757:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828005.620764:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828005.620781:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3409 00000100:00080000:2.0:1787828005.620799:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828005.620810:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828005.620873:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011637120/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828005.620952:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828005.620961:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828005.620965:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828005.620974:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3409 00000100:00080000:2.0:1787828005.620980:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828005.620988:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828005.621004:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828005.621027:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011637248/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828005.621048:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011637120:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828005.621050:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828005.621069:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828005.621074:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828005.621084:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3409 00000100:00080000:2.0:1787828005.621092:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828005.621101:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828005.621128:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828005.621154:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011637376/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828005.621169:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828005.621174:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828005.621176:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828005.621187:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3409 00000800:00000100:3.0:1787828005.621188:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787828005.621197:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828005.621206:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828005.621214:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828005.621232:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011637504/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828005.621290:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828005/real 1787828005] req@ffff907d9114bc00 x1874670011637120/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828005.621328:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011637120/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828005.621343:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828005.621349:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828005.621355:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011637120:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828005.621395:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011637248:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828005.621469:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828005.621499:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828005/real 1787828005] req@ffff907d911483c0 x1874670011637248/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828005.621526:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011637248/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828005.621545:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828005.621552:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828005.621560:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011637248:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828005.621573:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011637376:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828005.621660:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828005.621680:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828005/real 1787828005] req@ffff907d9114b480 x1874670011637376/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828005.621696:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011637376/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828005.621716:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828005.621721:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828005.621725:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011637376:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828005.621733:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011637504:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828005.621757:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828005.621770:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828005/real 1787828005] req@ffff907d91149680 x1874670011637504/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828005.621784:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011637504/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828005.621794:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828005.621798:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828005.621801:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011637504:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828005.748751:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828005.748873:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828006.772756:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828006.772921:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828007.796764:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828007.796904:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828008.820753:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828008.820899:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828009.844754:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828009.844863:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828010.740700:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828010.740710:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828010.740715:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828010.740726:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3414 00000100:00080000:2.0:1787828010.740740:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828010.740746:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828010.740790:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011637632/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828010.740816:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828010.740819:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828010.740820:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828010.740824:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3414 00000100:00080000:2.0:1787828010.740826:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828010.740828:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828010.740838:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828010.740850:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011637760/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828010.740858:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828010.740860:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828010.740862:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828010.740865:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3414 00000100:00080000:2.0:1787828010.740867:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828010.740869:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828010.740873:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828010.740878:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011637632:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828010.740883:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011637888/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828010.740891:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828010.740894:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828010.740895:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828010.740898:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3414 00000100:00080000:2.0:1787828010.740901:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828010.740903:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828010.740906:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828010.740914:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011638016/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828010.740951:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828010.741001:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828010/real 1787828010] req@ffff907d91149680 x1874670011637632/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828010.741018:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011637632/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828010.741028:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828010.741032:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828010.741035:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011637632:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828010.741042:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011637760:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828010.741059:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828010.741067:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828010/real 1787828010] req@ffff907d9114b480 x1874670011637760/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828010.741075:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011637760/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828010.741080:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828010.741082:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828010.741084:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011637760:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828010.741106:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011637888:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828010.741118:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828010.741124:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828010/real 1787828010] req@ffff907d911483c0 x1874670011637888/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828010.741132:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011637888/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828010.741137:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828010.741139:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828010.741141:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011637888:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828010.741145:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011638016:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828010.741154:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828010.741160:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828010/real 1787828010] req@ffff907d9114bc00 x1874670011638016/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828010.741168:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011638016/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828010.741174:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828010.741176:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828010.741178:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011638016:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828010.868684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828010.868737:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828011.892838:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828011.892939:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828012.916840:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828012.916946:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828013.940814:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828013.940958:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828014.964784:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828014.964870:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828015.860854:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828015.860867:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828015.860874:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828015.860889:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3419 00000100:00080000:2.0:1787828015.860906:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828015.860915:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828015.860973:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011638144/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828015.861011:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828015.861016:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828015.861019:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828015.861024:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3419 00000100:00080000:2.0:1787828015.861029:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828015.861034:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828015.861047:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828015.861068:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011638272/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828015.861080:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828015.861085:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828015.861088:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828015.861094:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3419 00000100:00080000:2.0:1787828015.861099:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828015.861106:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828015.861114:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828015.861131:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011638400/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828015.861146:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828015.861152:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828015.861155:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828015.861160:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3419 00000100:00080000:2.0:1787828015.861166:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828015.861171:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828015.861179:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828015.861198:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011638528/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828015.861735:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011638144:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787828015.861854:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828015.861925:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828015/real 1787828015] req@ffff907d9114a580 x1874670011638144/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828015.861951:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011638144/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828015.861968:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828015.861981:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828015.861986:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011638144:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828015.861996:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011638272:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828015.862126:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828015.862158:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828015/real 1787828015] req@ffff907d91149a40 x1874670011638272/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828015.862350:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011638272/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828015.862385:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828015.862407:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828015.862420:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011638272:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828015.862431:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011638400:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828015.862482:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828015.862501:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828015/real 1787828015] req@ffff907d9114ad00 x1874670011638400/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828015.862530:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011638400/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828015.862551:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828015.862558:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828015.862565:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011638400:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828015.862577:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011638528:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828015.862762:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828015.862782:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828015/real 1787828015] req@ffff907d9114b840 x1874670011638528/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828015.862796:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011638528/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828015.862803:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828015.862806:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828015.862810:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011638528:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828015.988770:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828015.988906:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828017.012794:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828017.012888:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828018.036755:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828018.036808:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828019.060772:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828019.060817:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828020.084716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828020.084790:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828020.980815:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828020.980831:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828020.980839:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828020.980860:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3424 00000100:00080000:0.0:1787828020.980882:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828020.980897:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828020.980975:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011638656/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828020.981022:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828020.981027:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828020.981030:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828020.981036:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3424 00000100:00080000:0.0:1787828020.981041:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828020.981047:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828020.981066:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828020.981087:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011638784/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828020.981102:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828020.981107:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828020.981111:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828020.981116:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3424 00000100:00080000:0.0:1787828020.981120:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828020.981133:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828020.981136:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011638656:192.168.203.104@tcp:250:kworker.0 00000008:00100000:0.0:1787828020.981141:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828020.981172:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011638912/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828020.981190:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828020.981195:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828020.981198:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828020.981205:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3424 00000100:00080000:0.0:1787828020.981211:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828020.981217:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828020.981227:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828020.981243:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011639040/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828020.981257:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828020.981361:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828020/real 1787828020] req@ffff907d90844f00 x1874670011638656/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828020.981392:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011638656/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828020.981408:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828020.981414:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828020.981419:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011638656:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828020.981431:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011638784:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828020.981465:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828020.981484:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828020/real 1787828020] req@ffff907d90845a40 x1874670011638784/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828020.981510:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011638784/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828020.981527:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828020.981533:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828020.981540:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011638784:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828020.981592:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011638912:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828020.981690:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828020.981713:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828020/real 1787828020] req@ffff907d90847480 x1874670011638912/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828020.981729:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011638912/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828020.981740:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828020.981745:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828020.981749:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011638912:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828020.981757:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011639040:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828020.981782:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828020.981802:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828020/real 1787828020] req@ffff907d90846940 x1874670011639040/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828020.981826:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011639040/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828020.981843:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828020.981850:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828020.981857:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011639040:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828021.108867:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828021.108924:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828022.132822:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828022.132974:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828023.156826:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828023.156898:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828024.180709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828024.180765:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828025.204704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828025.204916:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828026.100723:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828026.100732:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828026.100735:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828026.100744:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3429 00000100:00080000:0.0:1787828026.100754:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828026.100760:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828026.100792:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011639168/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828026.100815:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828026.100818:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828026.100820:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828026.100822:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3429 00000100:00080000:0.0:1787828026.100825:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828026.100827:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828026.100835:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828026.100845:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011639296/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828026.100853:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828026.100856:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828026.100859:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787828026.100861:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011639168:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828026.100864:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3429 00000100:00080000:0.0:1787828026.100870:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828026.100874:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828026.100877:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828026.100887:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011639424/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828026.100894:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828026.100896:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828026.100898:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828026.100900:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3429 00000100:00080000:0.0:1787828026.100902:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828026.100905:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828026.100908:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828026.100914:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011639552/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828026.100929:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828026.100965:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828026/real 1787828026] req@ffff907d90846d00 x1874670011639168/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828026.100977:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011639168/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828026.100985:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828026.100988:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828026.100991:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011639168:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828026.101010:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011639296:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828026.101047:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828026.101063:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828026/real 1787828026] req@ffff907d90845680 x1874670011639296/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828026.101081:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011639296/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828026.101088:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828026.101090:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828026.101092:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011639296:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828026.101097:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011639424:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828026.101114:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828026.101122:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828026/real 1787828026] req@ffff907d90844780 x1874670011639424/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828026.101130:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011639424/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828026.101140:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828026.101142:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828026.101145:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011639424:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828026.101148:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011639552:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828026.101158:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828026.101165:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828026/real 1787828026] req@ffff907d908452c0 x1874670011639552/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828026.101172:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011639552/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828026.101177:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828026.101179:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828026.101181:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011639552:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828026.228705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828026.228740:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828027.252812:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828027.252880:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828028.276734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828028.277009:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828029.300786:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828029.301257:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828030.324781:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828030.324858:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828031.220908:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828031.220918:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828031.220922:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828031.220938:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3434 00000100:00080000:2.0:1787828031.220948:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828031.220955:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828031.220996:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011639680/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828031.221031:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828031.221036:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828031.221038:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828031.221044:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3434 00000100:00080000:2.0:1787828031.221048:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828031.221062:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828031.221097:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828031.221118:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011639808/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828031.221134:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828031.221139:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828031.221141:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828031.221147:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3434 00000100:00080000:2.0:1787828031.221152:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828031.221161:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828031.221166:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011639680:192.168.203.104@tcp:250:kworker.0 00000008:00100000:2.0:1787828031.221170:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828031.221193:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011639936/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828031.221209:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828031.221213:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828031.221215:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828031.221219:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3434 00000100:00080000:2.0:1787828031.221223:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828031.221227:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828031.221232:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828031.221248:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011640064/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828031.221332:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828031.221399:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828031/real 1787828031] req@ffff907d911492c0 x1874670011639680/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828031.221422:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011639680/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828031.221440:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828031.221444:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828031.221449:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011639680:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828031.221456:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011639808:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828031.221483:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828031.221500:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828031/real 1787828031] req@ffff907d91148f00 x1874670011639808/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828031.221518:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011639808/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828031.221531:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828031.221536:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828031.221541:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011639808:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828031.221573:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011639936:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828031.221616:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828031.221673:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828031/real 1787828031] req@ffff907d9114a940 x1874670011639936/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828031.221684:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011639936/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828031.221691:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828031.221693:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828031.221696:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011639936:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828031.221702:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011640064:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828031.221737:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828031.221751:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828031/real 1787828031] req@ffff907d91148b40 x1874670011640064/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828031.221767:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011640064/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828031.221777:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828031.221779:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828031.221782:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011640064:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828031.348699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828031.348851:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828032.372742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828032.373083:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828033.396760:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828033.396846:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828034.420802:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828034.420900:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828035.444791:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828035.445021:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828036.340799:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828036.340813:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828036.340819:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828036.340834:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3440 00000100:00080000:2.0:1787828036.340864:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828036.340899:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828036.340954:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011640192/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828036.340997:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828036.341004:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828036.341008:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828036.341015:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3440 00000100:00080000:2.0:1787828036.341022:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828036.341029:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828036.341059:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828036.341079:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011640320/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828036.341095:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828036.341099:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828036.341102:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828036.341106:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3440 00000100:00080000:2.0:1787828036.341111:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828036.341117:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828036.341125:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828036.341142:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011640448/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828036.341158:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828036.341163:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828036.341166:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828036.341172:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3440 00000100:00080000:2.0:1787828036.341178:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787828036.341185:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011640192:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828036.341188:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828036.341202:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828036.341243:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011640576/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828036.341302:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828036.341353:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828036/real 1787828036] req@ffff907d91148b40 x1874670011640192/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828036.341380:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011640192/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828036.341403:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828036.341408:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828036.341415:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011640192:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828036.341428:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011640320:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828036.341515:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828036.341549:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828036/real 1787828036] req@ffff907d9114a940 x1874670011640320/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828036.341570:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011640320/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828036.341583:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828036.341588:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828036.341594:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011640320:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828036.341603:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011640448:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828036.341687:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828036.341714:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828036/real 1787828036] req@ffff907d91148f00 x1874670011640448/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828036.341735:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011640448/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828036.341748:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828036.341754:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828036.341760:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011640448:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828036.341796:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011640576:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828036.341829:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828036.341890:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828036/real 1787828036] req@ffff907d911492c0 x1874670011640576/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828036.341912:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011640576/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828036.341925:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828036.341929:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828036.341934:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011640576:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828036.468737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828036.468840:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828037.492721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828037.492846:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828038.516736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828038.516896:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828039.540747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828039.540812:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828040.564775:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828040.564847:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828041.460779:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828041.460795:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828041.460802:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828041.460819:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3445 00000100:00080000:0.0:1787828041.460837:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828041.460847:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828041.460922:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011640704/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828041.460967:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828041.460972:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828041.460975:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828041.460982:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3445 00000100:00080000:0.0:1787828041.460986:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828041.460991:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828041.461013:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828041.461041:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011640832/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828041.461067:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828041.461081:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011640704:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828041.461082:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828041.461097:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828041.461116:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3445 00000100:00080000:0.0:1787828041.461123:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828041.461131:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828041.461143:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828041.461164:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011640960/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828041.461207:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828041.461215:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828041.461219:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828041.461226:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3445 00000100:00080000:0.0:1787828041.461233:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787828041.461256:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787828041.461260:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828041.461270:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828041.461286:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011641088/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828041.461409:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828041/real 1787828041] req@ffff907d90844000 x1874670011640704/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828041.461449:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011640704/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828041.461471:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828041.461480:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828041.461489:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011640704:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828041.461531:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011640832:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828041.461571:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828041.461590:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828041/real 1787828041] req@ffff907d908443c0 x1874670011640832/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828041.461666:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011640832/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828041.461678:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828041.461683:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828041.461688:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011640832:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828041.461698:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011640960:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828041.461726:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828041.461741:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828041/real 1787828041] req@ffff907d90844b40 x1874670011640960/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828041.461755:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011640960/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828041.461766:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828041.461770:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828041.461774:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011640960:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828041.461781:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011641088:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828041.461799:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828041.461831:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828041/real 1787828041] req@ffff907d908470c0 x1874670011641088/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828041.461852:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011641088/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828041.461863:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828041.461867:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828041.461871:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011641088:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828041.588764:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828041.588915:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828042.612779:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828042.612850:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828043.636740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828043.636806:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828044.660730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828044.660780:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828045.684777:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828045.684826:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828046.580701:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828046.580714:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828046.580720:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828046.580743:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3450 00000100:00080000:0.0:1787828046.580758:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828046.580766:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828046.580811:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011641216/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828046.580843:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828046.580848:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828046.580851:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828046.580856:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3450 00000100:00080000:0.0:1787828046.580860:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828046.580864:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828046.580877:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828046.580893:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011641344/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828046.580910:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828046.580918:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1787828046.580920:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011641216:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828046.580921:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828046.580935:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3450 00000100:00080000:0.0:1787828046.580944:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828046.580950:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828046.580957:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828046.580977:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011641472/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828046.580989:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828046.580993:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828046.580996:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828046.581002:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3450 00000100:00080000:0.0:1787828046.581007:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828046.581011:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828046.581017:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828046.581033:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011641600/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828046.581100:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011641344:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828046.581145:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011641472:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828046.581216:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011641600:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828046.708770:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828046.708847:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828047.732718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828047.732835:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828048.756684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828048.756791:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787828049.652987:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787828049.653041:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787828049.653061:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828049.653118:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828049.653156:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828049.653188:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:3.0:1787828049.653265:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828046/real 1787828049] req@ffff907d90846580 x1874670011641344/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828049.653311:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011641344/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828049.653329:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828049.653336:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828049.653342:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011641344:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828049.653352:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828046/real 1787828049] req@ffff907d90847c00 x1874670011641472/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828049.653367:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011641472/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828049.653377:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828049.653381:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828049.653385:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011641472:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828049.653394:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828046/real 1787828049] req@ffff907d908470c0 x1874670011641600/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828049.653407:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011641600/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828049.653417:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828049.653421:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828049.653425:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011641600:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828049.653433:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828046/real 1787828049] req@ffff907d90845e00 x1874670011641216/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828049.653448:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011641216/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828049.653457:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828049.653460:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828049.653463:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011641216:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828049.780795:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828049.780888:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828050.804745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828050.804829:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828051.700759:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828051.700774:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828051.700781:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828051.700798:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3455 00000100:00080000:2.0:1787828051.700815:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828051.700825:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828051.700880:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011641728/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828051.700921:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828051.700926:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828051.700930:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828051.700937:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3455 00000100:00080000:2.0:1787828051.700944:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828051.700952:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828051.700972:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828051.701000:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011641856/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828051.701016:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011641728:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828051.701030:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828051.701041:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828051.701045:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828051.701069:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3455 00000100:00080000:2.0:1787828051.701079:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828051.701089:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828051.701102:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828051.701156:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011641984/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828051.701172:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787828051.701206:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828051.701214:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828051.701218:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828051.701228:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3455 00000100:00080000:2.0:1787828051.701237:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828051.701246:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828051.701260:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828051.701263:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828051/real 1787828051] req@ffff907d9114b840 x1874670011641728/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828051.701293:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011642112/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828051.701314:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011641728/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828051.701344:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828051.701354:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828051.701364:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011641728:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828051.701411:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011641856:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828051.701727:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828051.701761:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828051/real 1787828051] req@ffff907d9114ad00 x1874670011641856/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828051.701796:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011641856/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828051.701817:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828051.701826:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828051.701833:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011641856:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828051.701852:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011641984:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828051.702081:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828051.702319:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828051/real 1787828051] req@ffff907d91149a40 x1874670011641984/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828051.702345:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011641984/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828051.702358:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828051.702365:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828051.702371:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011641984:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828051.702386:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011642112:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828051.702424:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828051.702440:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828051/real 1787828051] req@ffff907d9114a580 x1874670011642112/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828051.702460:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011642112/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828051.702469:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828051.702473:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828051.702477:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011642112:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828051.828714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828051.828868:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828052.852774:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828052.852910:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828053.876725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828053.876913:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828054.900835:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828054.901017:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828055.924746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828055.924842:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828056.820714:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828056.820727:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828056.820732:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828056.820746:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3460 00000100:00080000:2.0:1787828056.820761:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828056.820770:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828056.820823:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011642240/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828056.820858:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828056.820863:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828056.820866:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828056.820870:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3460 00000100:00080000:2.0:1787828056.820873:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828056.820877:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828056.820888:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828056.820900:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011642368/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828056.820910:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828056.820913:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828056.820915:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828056.820919:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3460 00000100:00080000:2.0:1787828056.820921:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828056.820924:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828056.820928:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828056.820959:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011642496/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828056.820975:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828056.820980:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828056.820982:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828056.820991:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3460 00000100:00100000:3.0:1787828056.821000:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011642240:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828056.821001:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828056.821023:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828056.821038:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828056.821072:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011642624/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828056.821157:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828056.821246:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828056/real 1787828056] req@ffff907d9114bc00 x1874670011642240/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828056.821273:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011642240/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828056.821288:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828056.821294:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828056.821299:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011642240:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828056.821310:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011642368:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828056.821342:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828056.821358:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828056/real 1787828056] req@ffff907d911483c0 x1874670011642368/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828056.821390:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011642368/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828056.821408:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828056.821411:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828056.821415:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011642368:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828056.821443:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011642496:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828056.821486:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828056.821499:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828056/real 1787828056] req@ffff907d9114b480 x1874670011642496/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828056.821514:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011642496/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828056.821526:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828056.821532:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828056.821538:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011642496:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828056.821551:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011642624:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828056.821587:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828056.821609:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828056/real 1787828056] req@ffff907d91149680 x1874670011642624/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828056.821649:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011642624/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828056.821711:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828056.821715:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828056.821720:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011642624:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828056.948773:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828056.948884:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828057.972797:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828057.972906:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828058.996778:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828058.996921:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828060.020746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828060.020872:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828061.044742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828061.045030:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828061.940784:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828061.940797:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828061.940802:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828061.940814:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3465 00000100:00080000:1.0:1787828061.940826:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828061.940836:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828061.940880:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011642752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828061.940916:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828061.940921:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828061.940923:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828061.940928:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3465 00000100:00080000:1.0:1787828061.940932:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828061.940943:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828061.940955:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828061.940970:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011642880/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828061.940991:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828061.940995:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828061.940997:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828061.941002:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3465 00000100:00080000:1.0:1787828061.941005:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828061.941013:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828061.941016:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011642752:192.168.203.104@tcp:250:kworker.0 00000008:00100000:1.0:1787828061.941020:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828061.941037:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011643008/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828061.941071:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828061.941076:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828061.941079:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828061.941083:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3465 00000100:00080000:1.0:1787828061.941088:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828061.941092:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828061.941098:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828061.941112:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011643136/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828061.941131:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828061.941209:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828061/real 1787828061] req@ffff907d908a2580 x1874670011642752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828061.941246:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011642752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828061.941266:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828061.941274:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828061.941282:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011642752:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828061.941297:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011642880:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828061.941331:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828061.941348:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828061/real 1787828061] req@ffff907d908a21c0 x1874670011642880/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828061.941362:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011642880/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828061.941373:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828061.941377:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828061.941381:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011642880:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828061.941407:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011643008:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828061.941428:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828061.941441:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828061/real 1787828061] req@ffff907d908a0f00 x1874670011643008/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828061.941454:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011643008/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828061.941463:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828061.941467:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828061.941470:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011643008:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828061.941476:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011643136:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828061.941492:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828061.941503:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828061/real 1787828061] req@ffff907d908a12c0 x1874670011643136/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828061.941515:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011643136/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828061.941524:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828061.941527:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828061.941530:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011643136:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828062.068723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828062.068861:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828063.092683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828063.092883:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828064.116698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828064.116911:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828065.140704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828065.140908:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828066.164707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828066.164877:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828067.060722:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828067.060735:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828067.060741:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828067.060753:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3470 00000100:00080000:1.0:1787828067.060767:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828067.060777:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828067.060829:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011643264/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828067.060879:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828067.060886:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828067.060890:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828067.060898:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3470 00000100:00080000:1.0:1787828067.060905:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828067.060913:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828067.060930:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828067.060958:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011643392/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828067.060976:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828067.060981:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828067.060984:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828067.060989:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3470 00000100:00080000:1.0:1787828067.060994:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828067.061000:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828067.061035:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828067.061065:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011643520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828067.061069:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011643264:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828067.061113:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828067.061118:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828067.061121:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828067.061131:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3470 00000100:00080000:1.0:1787828067.061138:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828067.061144:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828067.061151:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828067.061173:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011643648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828067.061225:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828067.061314:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828067/real 1787828067] req@ffff907d908a3840 x1874670011643264/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828067.061347:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011643264/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828067.061366:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828067.061374:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828067.061381:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011643264:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828067.061392:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011643392:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828067.061422:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828067.061438:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828067/real 1787828067] req@ffff907d908a1680 x1874670011643392/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828067.061453:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011643392/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828067.061463:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828067.061467:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828067.061471:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011643392:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828067.061511:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011643520:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828067.061551:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828067.061564:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828067/real 1787828067] req@ffff907d908a12c0 x1874670011643520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828067.061579:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011643520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828067.061590:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828067.061594:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828067.061598:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011643520:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828067.061657:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011643648:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828067.061684:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828067.061698:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828067/real 1787828067] req@ffff907d908a0f00 x1874670011643648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828067.061713:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011643648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828067.061723:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828067.061727:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828067.061731:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011643648:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828067.188754:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828067.188882:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828068.212746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828068.212953:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828069.236752:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828069.236832:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828070.260780:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828070.260919:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828071.284715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828071.284898:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828072.180827:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828072.180843:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828072.180850:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828072.180866:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3475 00000100:00080000:2.0:1787828072.180883:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828072.180896:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828072.180960:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011643776/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828072.181033:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828072.181039:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828072.181043:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828072.181050:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3475 00000100:00080000:2.0:1787828072.181056:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828072.181061:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828072.181086:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828072.181106:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011643904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828072.181121:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828072.181127:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828072.181129:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828072.181140:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3475 00000100:00100000:0.0:1787828072.181141:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011643776:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828072.181145:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828072.181156:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828072.181170:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828072.181183:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011644032/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828072.181195:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828072.181200:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828072.181203:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828072.181207:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3475 00000100:00080000:2.0:1787828072.181211:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828072.181215:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828072.181221:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828072.181235:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011644160/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828072.181297:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828072.181377:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828072/real 1787828072] req@ffff907d91149680 x1874670011643776/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828072.181415:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011643776/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828072.181439:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828072.181447:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828072.181456:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011643776:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828072.181501:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011643904:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828072.181554:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828072.181583:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828072/real 1787828072] req@ffff907d9114b480 x1874670011643904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828072.181690:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011643904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828072.181710:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828072.181718:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828072.181726:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011643904:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828072.181739:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011644032:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828072.181784:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828072.181808:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828072/real 1787828072] req@ffff907d911483c0 x1874670011644032/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828072.181832:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011644032/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828072.181850:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828072.181856:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828072.181862:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011644032:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828072.181875:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011644160:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828072.181911:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828072.181933:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828072/real 1787828072] req@ffff907d9114bc00 x1874670011644160/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828072.182003:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011644160/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828072.182020:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828072.182027:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828072.182034:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011644160:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828072.308736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828072.308849:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828073.332706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828073.332785:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828074.356758:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828074.356886:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828075.380733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828075.380836:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828076.404797:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828076.404841:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828077.300832:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828077.300848:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828077.300855:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828077.300871:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3481 00000100:00080000:0.0:1787828077.300889:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828077.300900:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828077.300996:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011644288/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828077.301049:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828077.301054:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828077.301057:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828077.301063:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3481 00000100:00080000:0.0:1787828077.301067:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828077.301072:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828077.301088:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828077.301116:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011644416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828077.301143:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828077.301151:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828077.301157:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787828077.301160:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011644288:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828077.301165:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3481 00000100:00080000:0.0:1787828077.301180:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828077.301200:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828077.301207:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828077.301228:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011644544/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828077.301240:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828077.301246:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828077.301250:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828077.301254:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3481 00000100:00080000:0.0:1787828077.301263:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828077.301269:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828077.301277:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787828077.301296:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828077.301307:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011644672/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828077.301386:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828077/real 1787828077] req@ffff907d90844b40 x1874670011644288/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828077.301415:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011644288/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828077.301431:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828077.301437:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828077.301443:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011644288:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828077.301476:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011644416:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828077.301522:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828077.301536:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828077/real 1787828077] req@ffff907d908443c0 x1874670011644416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828077.301551:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011644416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828077.301560:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828077.301563:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828077.301567:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011644416:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828077.301574:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011644544:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828077.301591:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828077.301601:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828077/real 1787828077] req@ffff907d90844000 x1874670011644544/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828077.301613:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011644544/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828077.301621:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828077.301625:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828077.301628:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011644544:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828077.301634:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011644672:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828077.301687:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828077.301698:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828077/real 1787828077] req@ffff907d908452c0 x1874670011644672/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828077.301711:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011644672/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828077.301732:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828077.301738:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828077.301744:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011644672:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828077.428797:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828077.428956:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828078.452819:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828078.452909:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828079.476790:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828079.476899:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828080.500793:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828080.500907:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828081.524743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828081.525174:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828082.420852:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828082.420866:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828082.420871:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828082.420884:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3486 00000100:00080000:2.0:1787828082.420942:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828082.420952:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828082.421001:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011644800/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828082.421042:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828082.421046:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828082.421048:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828082.421053:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3486 00000100:00080000:2.0:1787828082.421057:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828082.421061:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828082.421072:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828082.421087:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011644928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828082.421103:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828082.421107:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828082.421109:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828082.421112:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3486 00000100:00080000:2.0:1787828082.421116:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828082.421119:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828082.421125:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828082.421136:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011645056/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828082.421148:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828082.421156:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828082.421160:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787828082.421160:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011644800:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828082.421185:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3486 00000100:00080000:2.0:1787828082.421189:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828082.421194:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828082.421199:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828082.421215:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011645184/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828082.421299:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828082.421386:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828082/real 1787828082] req@ffff907d9114a580 x1874670011644800/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828082.421426:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011644800/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828082.421450:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828082.421460:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828082.421469:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011644800:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828082.421485:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011644928:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828082.421533:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828082.421555:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828082/real 1787828082] req@ffff907d91149a40 x1874670011644928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828082.421572:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011644928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828082.421584:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828082.421589:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828082.421593:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011644928:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828082.421682:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011645056:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828082.421714:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828082.421729:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828082/real 1787828082] req@ffff907d9114ad00 x1874670011645056/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828082.421744:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011645056/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828082.421755:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828082.421759:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828082.421763:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011645056:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828082.421770:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011645184:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828082.421801:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828082.421812:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828082/real 1787828082] req@ffff907d9114b840 x1874670011645184/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828082.421826:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011645184/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828082.421836:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828082.421839:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828082.421843:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011645184:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828082.548741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828082.548918:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828083.572823:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828083.572835:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828084.596700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828084.596783:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828085.620781:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828085.620807:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828086.644778:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828086.644918:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828087.540981:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828087.540995:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828087.541002:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828087.541015:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3491 00000100:00080000:0.0:1787828087.541032:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828087.541059:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828087.541104:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011645312/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828087.541140:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828087.541145:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828087.541147:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828087.541152:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3491 00000100:00080000:0.0:1787828087.541156:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828087.541160:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828087.541173:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828087.541195:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011645440/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828087.541240:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828087.541241:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011645312:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828087.541266:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828087.541274:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828087.541288:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3491 00000100:00080000:0.0:1787828087.541298:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828087.541308:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828087.541322:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828087.541370:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011645568/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828087.541383:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828087.541389:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828087.541391:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828087.541402:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3491 00000800:00000100:2.0:1787828087.541404:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787828087.541413:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828087.541442:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828087.541453:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828087.541475:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011645696/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828087.541496:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828087/real 1787828087] req@ffff907d90844780 x1874670011645312/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828087.541530:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011645312/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828087.541551:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828087.541558:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828087.541565:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011645312:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828087.541672:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011645440:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828087.541748:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828087.541788:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828087/real 1787828087] req@ffff907d90845680 x1874670011645440/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828087.541806:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011645440/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828087.541828:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828087.541834:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828087.541840:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011645440:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828087.541855:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011645568:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828087.541944:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828087.541979:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828087/real 1787828087] req@ffff907d90846d00 x1874670011645568/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828087.542002:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011645568/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828087.542013:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828087.542017:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828087.542020:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011645568:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828087.542029:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011645696:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828087.542076:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828087.542090:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828087/real 1787828087] req@ffff907d90846940 x1874670011645696/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828087.542104:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011645696/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828087.542115:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828087.542119:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828087.542123:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011645696:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787828087.668805:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828087.668806:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787828088.692781:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828088.692974:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828089.716843:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828089.716876:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828090.740805:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828090.740887:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828091.764889:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828091.764926:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828092.660769:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828092.660777:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828092.660782:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828092.660790:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3496 00000100:00080000:2.0:1787828092.660800:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828092.660806:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828092.660855:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011645824/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828092.660881:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828092.660884:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828092.660885:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828092.660888:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3496 00000100:00080000:2.0:1787828092.660890:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828092.660893:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828092.660901:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828092.660911:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011645952/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828092.660919:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828092.660922:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828092.660925:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787828092.660925:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011645824:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828092.660933:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3496 00000100:00080000:2.0:1787828092.660936:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828092.660939:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828092.660942:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828092.660951:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011646080/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828092.660966:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828092.660970:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828092.660972:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828092.660976:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3496 00000100:00080000:2.0:1787828092.660980:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828092.660984:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:3.0:1787828092.660989:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:2.0:1787828092.660992:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828092.661016:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011646208/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828092.661026:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828092/real 1787828092] req@ffff907d911492c0 x1874670011645824/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828092.661042:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011645824/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828092.661051:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828092.661054:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828092.661057:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011645824:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828092.661075:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011645952:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828092.661089:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828092.661097:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828092/real 1787828092] req@ffff907d91148f00 x1874670011645952/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828092.661110:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011645952/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828092.661119:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828092.661122:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828092.661125:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011645952:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828092.661131:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011646080:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828092.661151:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828092.661162:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828092/real 1787828092] req@ffff907d9114a940 x1874670011646080/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828092.661179:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011646080/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828092.661185:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828092.661188:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828092.661191:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011646080:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828092.661197:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011646208:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828092.661218:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828092.661230:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828092/real 1787828092] req@ffff907d91148b40 x1874670011646208/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828092.661242:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011646208/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828092.661250:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828092.661253:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828092.661256:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011646208:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828092.788726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828092.788780:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828093.812756:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828093.812958:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828094.836788:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828094.836897:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828095.860781:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828095.860934:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828096.884708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828096.884777:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828097.780772:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828097.780786:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828097.780817:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828097.780836:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3501 00000100:00080000:2.0:1787828097.780856:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828097.780870:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828097.780934:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011646336/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828097.780984:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828097.780995:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828097.781000:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828097.781010:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3501 00000100:00080000:2.0:1787828097.781020:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828097.781029:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828097.781046:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828097.781074:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011646336:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828097.781090:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011646464/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828097.781119:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828097.781126:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828097.781130:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828097.781138:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3501 00000100:00080000:2.0:1787828097.781168:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828097.781179:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:3.0:1787828097.781192:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:2.0:1787828097.781196:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828097.781229:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011646592/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828097.781250:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828097.781258:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828097.781263:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787828097.781275:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828097/real 1787828097] req@ffff907d91148b40 x1874670011646336/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828097.781276:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3501 00000100:00080000:2.0:1787828097.781288:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828097.781303:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828097.781325:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828097.781340:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011646336/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828097.781357:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011646720/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828097.781371:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828097.781390:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828097.781402:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011646336:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828097.781444:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011646464:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828097.781491:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828097.781516:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828097/real 1787828097] req@ffff907d9114a940 x1874670011646464/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828097.781540:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011646464/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828097.781555:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828097.781561:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828097.781567:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011646464:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828097.781601:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011646592:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828097.781735:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828097.781769:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828097/real 1787828097] req@ffff907d91148f00 x1874670011646592/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828097.781829:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011646592/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828097.781848:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828097.781855:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828097.781862:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011646592:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828097.781877:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011646720:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828097.781919:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828097.781945:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828097/real 1787828097] req@ffff907d911492c0 x1874670011646720/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828097.781968:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011646720/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828097.781983:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828097.781989:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828097.781995:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011646720:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828097.908829:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828097.908838:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828098.932737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828098.932774:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828099.956733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828099.956905:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828100.980705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828100.980740:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828102.004751:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828102.004863:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828102.900804:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828102.900820:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828102.900827:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828102.900843:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3506 00000100:00080000:0.0:1787828102.900862:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828102.900872:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828102.900929:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011646848/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828102.900972:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828102.900978:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828102.900981:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828102.900986:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3506 00000100:00080000:0.0:1787828102.900990:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828102.900995:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828102.901012:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828102.901028:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011646976/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828102.901046:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828102.901058:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1787828102.901062:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011646848:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828102.901065:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828102.901086:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3506 00000100:00080000:0.0:1787828102.901091:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828102.901096:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828102.901104:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828102.901121:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011647104/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828102.901134:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828102.901140:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828102.901143:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828102.901148:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3506 00000100:00080000:0.0:1787828102.901157:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828102.901163:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828102.901169:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828102.901186:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011647232/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828102.901196:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828102.901264:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828102/real 1787828102] req@ffff907d90847480 x1874670011646848/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828102.901293:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011646848/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828102.901309:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828102.901315:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828102.901320:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011646848:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828102.901352:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011646976:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828102.901385:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828102.901402:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828102/real 1787828102] req@ffff907d90845a40 x1874670011646976/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828102.901417:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011646976/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828102.901428:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828102.901432:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828102.901437:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011646976:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828102.901445:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011647104:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828102.901466:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828102.901477:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828102/real 1787828102] req@ffff907d90844f00 x1874670011647104/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828102.901492:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011647104/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828102.901503:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828102.901507:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828102.901511:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011647104:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828102.901518:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011647232:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828102.901536:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828102.901568:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828102/real 1787828102] req@ffff907d90846940 x1874670011647232/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828102.901594:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011647232/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828102.901640:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828102.901645:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828102.901649:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011647232:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828103.028740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828103.028915:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828104.052745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828104.052882:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828105.076784:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828105.076870:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828106.100769:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828106.100870:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828107.124841:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828107.124877:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828108.020875:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828108.020888:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828108.020894:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828108.020909:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3511 00000100:00080000:3.0:1787828108.020924:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828108.020934:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828108.020992:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011647360/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828108.021045:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828108.021050:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828108.021053:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828108.021058:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3511 00000100:00080000:3.0:1787828108.021062:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828108.021066:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828108.021079:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828108.021094:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011647488/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828108.021116:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828108.021120:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828108.021123:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828108.021127:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3511 00000100:00080000:3.0:1787828108.021130:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828108.021135:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828108.021141:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828108.021153:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011647616/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828108.021175:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828108.021196:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011647360:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828108.021196:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828108.021204:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828108.021217:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3511 00000100:00080000:3.0:1787828108.021221:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828108.021236:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828108.021243:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828108.021260:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011647744/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828108.021322:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828108.021391:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828108/real 1787828108] req@ffff907da562c3c0 x1874670011647360/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828108.021441:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011647360/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828108.021458:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828108.021463:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828108.021469:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011647360:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828108.021479:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011647488:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828108.021519:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828108.021537:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828108/real 1787828108] req@ffff907da562d2c0 x1874670011647488/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828108.021553:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011647488/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828108.021564:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828108.021568:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828108.021572:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011647488:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828108.021579:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011647616:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828108.021599:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828108.021673:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828108/real 1787828108] req@ffff907da562e580 x1874670011647616/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828108.021691:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011647616/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828108.021702:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828108.021707:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828108.021711:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011647616:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828108.021773:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011647744:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828108.021814:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828108.021829:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828108/real 1787828108] req@ffff907da562c000 x1874670011647744/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828108.021843:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011647744/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828108.021864:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828108.021868:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828108.021872:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011647744:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828108.148803:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828108.148936:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828109.172674:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828109.172759:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828110.196802:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828110.196901:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828111.220771:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828111.220858:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828112.244783:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828112.244853:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828113.140792:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828113.140803:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828113.140808:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828113.140819:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3516 00000100:00080000:2.0:1787828113.140831:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828113.140839:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828113.140890:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011647872/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828113.140924:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828113.140928:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828113.140930:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828113.140950:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3516 00000100:00080000:2.0:1787828113.140956:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828113.140963:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828113.140981:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828113.141010:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011648000/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828113.141034:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011647872:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828113.141058:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828113.141066:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828113.141069:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828113.141077:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3516 00000100:00080000:2.0:1787828113.141084:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828113.141091:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828113.141101:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828113.141119:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011648128/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828113.141149:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828113.141152:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828113.141155:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828113.141159:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3516 00000100:00080000:2.0:1787828113.141163:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828113.141167:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828113.141174:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828113.141193:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011648256/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828113.141200:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011648000:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828113.141286:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011648128:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828113.141315:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011648256:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787828113.268855:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828113.268858:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787828114.292730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828114.292867:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828115.316797:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828115.316834:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787828116.213156:0:2337:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:2.0:1787828116.213212:0:2337:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787828116.213230:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787828116.213295:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787828116.213308:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:2.0:1787828116.213319:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:0.0:1787828116.213372:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828113/real 1787828116] req@ffff907d9114a580 x1874670011648256/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828116.213416:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011648256/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828116.213443:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828116.213451:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828116.213459:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011648256:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828116.213472:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828113/real 1787828116] req@ffff907d9114ad00 x1874670011648000/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828116.213490:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011648000/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828116.213501:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828116.213506:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828116.213511:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011648000:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828116.213520:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828113/real 1787828116] req@ffff907d91149a40 x1874670011648128/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828116.213534:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011648128/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828116.213545:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828116.213549:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828116.213554:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011648128:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828116.213564:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828113/real 1787828116] req@ffff907d9114b840 x1874670011647872/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828116.213580:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011647872/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828116.213591:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828116.213595:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828116.213599:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011647872:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828116.340786:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828116.340853:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828117.364830:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828117.364945:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828118.260763:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828118.260776:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828118.260781:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828118.260795:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3522 00000100:00080000:2.0:1787828118.260822:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828118.260832:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828118.260878:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011648384/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828118.260911:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828118.260917:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828118.260919:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828118.260924:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3522 00000100:00080000:2.0:1787828118.260929:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828118.260934:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828118.260947:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828118.260965:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011648512/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828118.260978:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011648384:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828118.260986:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828118.260993:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828118.260996:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828118.261002:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3522 00000100:00080000:2.0:1787828118.261006:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828118.261011:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828118.261018:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828118.261037:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011648640/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828118.261049:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828118.261053:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828118.261057:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828118.261062:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3522 00000100:00080000:2.0:1787828118.261066:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828118.261072:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:0.0:1787828118.261081:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:2.0:1787828118.261083:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828118.261104:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011648768/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828118.261154:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828118/real 1787828118] req@ffff907d9114bc00 x1874670011648384/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828118.261190:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011648384/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828118.261211:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828118.261219:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828118.261227:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011648384:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828118.261266:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011648512:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828118.261315:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828118.261343:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828118/real 1787828118] req@ffff907d911483c0 x1874670011648512/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828118.261367:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011648512/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828118.261384:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828118.261405:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828118.261413:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011648512:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828118.261428:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011648640:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828118.261469:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828118.261492:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828118/real 1787828118] req@ffff907d9114b480 x1874670011648640/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828118.261514:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011648640/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828118.261530:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828118.261536:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828118.261542:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011648640:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828118.261554:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011648768:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828118.261647:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828118.261709:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828118/real 1787828118] req@ffff907d91149680 x1874670011648768/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828118.261736:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011648768/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828118.261749:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828118.261756:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828118.261763:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011648768:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828118.388781:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828118.388841:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828119.412768:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828119.412859:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828120.436780:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828120.436872:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828121.460759:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828121.460907:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828122.484761:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828122.484896:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828123.380797:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828123.380825:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828123.380832:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828123.380849:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3527 00000100:00080000:2.0:1787828123.380867:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828123.380879:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828123.380952:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011648896/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828123.380995:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828123.381012:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828123.381015:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828123.381025:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3527 00000100:00080000:2.0:1787828123.381033:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828123.381039:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828123.381056:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828123.381066:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011648896:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828123.381081:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011649024/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828123.381100:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828123.381106:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828123.381110:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828123.381118:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3527 00000100:00080000:2.0:1787828123.381134:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787828123.381143:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787828123.381148:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828123.381159:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828123.381174:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011649152/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828123.381186:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828123.381190:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828123.381192:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828123.381197:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3527 00000100:00080000:2.0:1787828123.381201:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828123.381205:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828123.381210:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828123.381221:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011649280/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828123.381286:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828123/real 1787828123] req@ffff907d91149680 x1874670011648896/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828123.381404:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011648896/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828123.381487:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828123.381493:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828123.381497:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011648896:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828123.381522:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011649024:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828123.381563:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828123.381588:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828123/real 1787828123] req@ffff907d9114b480 x1874670011649024/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828123.381600:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011649024/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828123.381671:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828123.381677:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828123.381685:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011649024:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828123.381692:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011649152:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828123.381721:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828123.381731:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828123/real 1787828123] req@ffff907d911483c0 x1874670011649152/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828123.381739:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011649152/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828123.381745:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828123.381747:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828123.381749:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011649152:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828123.381753:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011649280:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828123.381764:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828123.381788:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828123/real 1787828123] req@ffff907d9114bc00 x1874670011649280/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828123.381810:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011649280/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828123.381820:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828123.381823:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828123.381827:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011649280:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828123.508736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828123.508775:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828124.532703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828124.532933:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828125.556719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828125.556873:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828126.580779:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828126.580806:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828127.604714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828127.604826:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828128.500734:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828128.500748:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828128.500754:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828128.500767:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3532 00000100:00080000:2.0:1787828128.500780:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828128.500789:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828128.500835:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011649408/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828128.500870:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828128.500875:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828128.500878:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828128.500883:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3532 00000100:00080000:2.0:1787828128.500887:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828128.500891:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828128.500903:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828128.500919:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011649536/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828128.500954:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828128.500958:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828128.500961:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787828128.500970:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011649408:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828128.500972:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3532 00000100:00080000:2.0:1787828128.500979:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828128.500984:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828128.500990:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828128.501006:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011649664/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828128.501019:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828128.501024:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828128.501026:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828128.501031:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3532 00000100:00080000:2.0:1787828128.501036:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828128.501040:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828128.501046:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828128.501059:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011649792/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828128.501084:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828128.501159:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828128/real 1787828128] req@ffff907d9114b840 x1874670011649408/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828128.501183:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011649408/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828128.501197:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828128.501202:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828128.501208:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011649408:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828128.501236:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011649536:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828128.501265:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828128.501280:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828128/real 1787828128] req@ffff907d91149a40 x1874670011649536/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828128.501297:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011649536/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828128.501307:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828128.501311:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828128.501315:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011649536:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828128.501322:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011649664:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828128.501341:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828128.501353:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828128/real 1787828128] req@ffff907d9114ad00 x1874670011649664/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828128.501367:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011649664/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828128.501377:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828128.501380:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828128.501384:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011649664:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828128.501390:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011649792:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828128.501408:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828128.501418:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828128/real 1787828128] req@ffff907d9114a580 x1874670011649792/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828128.501432:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011649792/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828128.501458:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828128.501462:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828128.501466:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011649792:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828128.628718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828128.628966:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828129.652719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828129.652817:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828130.676749:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828130.676775:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828131.700708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828131.700771:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828132.724725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828132.724843:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828133.620748:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828133.620760:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828133.620766:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828133.620777:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3537 00000100:00080000:2.0:1787828133.620789:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828133.620814:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828133.620850:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011649920/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828133.620904:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828133.620909:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828133.620911:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828133.620916:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3537 00000100:00080000:2.0:1787828133.620921:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828133.620926:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828133.620937:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828133.620949:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011649920:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828133.620956:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011650048/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828133.620970:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828133.620974:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828133.620976:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828133.620980:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3537 00000100:00080000:2.0:1787828133.620984:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828133.620987:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828133.620992:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828133.621005:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011650176/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828133.621023:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828133.621027:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828133.621029:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828133.621032:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3537 00000100:00080000:2.0:1787828133.621036:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787828133.621039:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787828133.621045:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828133.621051:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828133.621063:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011650304/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828133.621095:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828133/real 1787828133] req@ffff907d911492c0 x1874670011649920/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828133.621116:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011649920/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828133.621128:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828133.621132:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828133.621137:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011649920:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828133.621161:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011650048:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828133.621184:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828133.621213:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828133/real 1787828133] req@ffff907d91148f00 x1874670011650048/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828133.621223:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011650048/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828133.621230:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828133.621232:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828133.621235:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011650048:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828133.621239:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011650176:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828133.621252:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828133.621259:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828133/real 1787828133] req@ffff907d9114a940 x1874670011650176/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828133.621267:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011650176/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828133.621274:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828133.621276:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828133.621278:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011650176:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828133.621281:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011650304:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828133.621292:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828133.621298:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828133/real 1787828133] req@ffff907d91148b40 x1874670011650304/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828133.621306:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011650304/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828133.621312:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828133.621314:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828133.621316:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011650304:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828133.748730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828133.749015:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828134.772752:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828134.772840:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787828135.796706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828135.796804:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828136.820881:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828136.820976:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828137.844763:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828137.844837:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828138.740726:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828138.740740:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828138.740745:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828138.740758:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3542 00000100:00080000:2.0:1787828138.740772:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828138.740781:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828138.740841:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011650432/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828138.740887:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828138.740892:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828138.740894:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828138.740899:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3542 00000100:00080000:2.0:1787828138.740904:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828138.740909:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828138.740920:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828138.740935:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011650560/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828138.740950:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828138.740973:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828138.740977:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828138.740987:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3542 00000100:00080000:2.0:1787828138.740996:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828138.741004:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828138.741017:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828138.741017:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011650432:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828138.741063:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011650688/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828138.741088:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828138.741094:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828138.741099:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828138.741106:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3542 00000100:00080000:2.0:1787828138.741123:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787828138.741129:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787828138.741153:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828138.741167:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828138.741183:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828138/real 1787828138] req@ffff907d91148b40 x1874670011650432/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828138.741197:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011650816/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828138.741217:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011650432/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828138.741233:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828138.741241:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828138.741249:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011650432:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828138.741259:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011650560:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828138.741287:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828138.741305:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828138/real 1787828138] req@ffff907d9114a940 x1874670011650560/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828138.741318:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011650560/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828138.741328:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828138.741332:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828138.741336:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011650560:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828138.741364:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011650688:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828138.741396:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828138.741416:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828138/real 1787828138] req@ffff907d91148f00 x1874670011650688/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828138.741439:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011650688/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828138.741455:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828138.741461:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828138.741467:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011650688:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828138.741479:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011650816:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828138.741538:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828138.741558:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828138/real 1787828138] req@ffff907d911492c0 x1874670011650816/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828138.741573:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011650816/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828138.741583:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828138.741587:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828138.741590:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011650816:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828138.868757:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828138.868856:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828139.892736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828139.892862:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828140.916691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828140.916734:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828141.940766:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828141.940856:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828142.964733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828142.964813:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828143.860710:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828143.860723:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828143.860729:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828143.860757:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3547 00000100:00080000:2.0:1787828143.860771:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828143.860781:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828143.860831:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011650944/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828143.860866:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828143.860871:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828143.860874:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828143.860879:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3547 00000100:00080000:2.0:1787828143.860883:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828143.860887:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828143.860899:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828143.860915:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011651072/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828143.860930:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828143.860934:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828143.860938:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828143.860943:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3547 00000100:00080000:2.0:1787828143.860947:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828143.860956:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828143.860962:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011650944:192.168.203.104@tcp:250:kworker.0 00000008:00100000:2.0:1787828143.860964:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828143.860981:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011651200/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828143.860995:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828143.860999:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828143.861002:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828143.861006:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3547 00000100:00080000:2.0:1787828143.861010:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828143.861014:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828143.861021:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828143.861036:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011651328/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828143.861068:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828143.861135:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828143/real 1787828143] req@ffff907d9114a580 x1874670011650944/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828143.861162:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011650944/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828143.861176:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828143.861181:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828143.861187:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011650944:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828143.861197:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011651072:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828143.861225:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828143.861241:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828143/real 1787828143] req@ffff907d9114ad00 x1874670011651072/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828143.861256:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011651072/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828143.861267:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828143.861273:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828143.861280:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011651072:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828143.861320:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011651200:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828143.861390:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828143.861416:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828143/real 1787828143] req@ffff907d91149a40 x1874670011651200/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828143.861441:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011651200/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828143.861459:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828143.861466:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828143.861501:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011651200:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828143.861515:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011651328:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828143.861559:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828143.861585:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828143/real 1787828143] req@ffff907d9114b840 x1874670011651328/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828143.861668:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011651328/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828143.861681:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828143.861685:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828143.861703:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011651328:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828143.988752:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828143.988863:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828145.012688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828145.012705:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828146.036800:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828146.036811:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828147.060822:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828147.060974:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828148.084808:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828148.084912:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828148.980789:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828148.980806:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828148.980814:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828148.980831:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3552 00000100:00080000:3.0:1787828148.980849:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828148.980860:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828148.980922:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011651456/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828148.980969:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828148.980975:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828148.980977:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828148.980983:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3552 00000100:00080000:3.0:1787828148.980987:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828148.980992:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828148.981008:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828148.981024:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011651584/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828148.981038:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828148.981043:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828148.981045:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828148.981049:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3552 00000100:00080000:3.0:1787828148.981053:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828148.981057:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828148.981063:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828148.981075:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011651712/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828148.981097:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828148.981103:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828148.981106:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828148.981110:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3552 00000100:00080000:3.0:1787828148.981114:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828148.981118:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828148.981129:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011651456:192.168.203.104@tcp:250:kworker.0 00000008:00100000:3.0:1787828148.981129:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828148.981151:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011651840/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828148.981272:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828148.981361:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828148/real 1787828148] req@ffff907da562f480 x1874670011651456/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828148.981390:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011651456/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828148.981406:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828148.981412:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828148.981421:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011651456:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828148.981433:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011651584:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828148.981504:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828148.981523:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828148/real 1787828148] req@ffff907da562c780 x1874670011651584/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828148.981539:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011651584/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828148.981551:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828148.981555:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828148.981559:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011651584:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828148.981567:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011651712:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828148.981589:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828148.981601:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828148/real 1787828148] req@ffff907da562ed00 x1874670011651712/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828148.981662:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011651712/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828148.981682:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828148.981687:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828148.981691:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011651712:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828148.981730:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011651840:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828148.981756:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828148.981770:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828148/real 1787828148] req@ffff907da562cf00 x1874670011651840/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828148.981784:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011651840/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828148.981794:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828148.981798:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828148.981802:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011651840:192.168.203.104@tcp:8:kworker.0 00010000:00010000:0.0:1787828149.108839:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828149.108913:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787828150.132766:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828150.132860:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828151.156741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828151.156961:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828152.180780:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828152.180912:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828153.204711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828153.205042:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828154.100763:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828154.100778:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828154.100783:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828154.100794:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3557 00000100:00080000:0.0:1787828154.100808:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828154.100816:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828154.100861:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011651968/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828154.100897:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828154.100902:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828154.100904:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828154.100909:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3557 00000100:00080000:0.0:1787828154.100914:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828154.100919:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828154.100930:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828154.100940:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011651968:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828154.100944:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011652096/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828154.100957:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828154.100960:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828154.100961:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828154.100964:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3557 00000100:00080000:0.0:1787828154.100967:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828154.100970:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828154.100973:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828154.100981:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011652224/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828154.100988:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828154.100991:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828154.100992:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828154.100994:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3557 00000100:00080000:0.0:1787828154.100996:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828154.100999:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828154.101003:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828154.101011:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011652352/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828154.101032:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828154.101082:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828154/real 1787828154] req@ffff907d90846d00 x1874670011651968/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828154.101101:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011651968/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828154.101112:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828154.101116:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828154.101120:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011651968:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828154.101148:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011652096:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828154.101169:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828154.101181:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828154/real 1787828154] req@ffff907d90845680 x1874670011652096/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828154.101192:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011652096/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828154.101200:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828154.101203:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828154.101206:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011652096:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828154.101211:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011652224:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828154.101227:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828154.101236:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828154/real 1787828154] req@ffff907d90844780 x1874670011652224/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828154.101246:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011652224/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828154.101254:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828154.101257:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828154.101260:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011652224:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828154.101265:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011652352:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828154.101279:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828154.101287:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828154/real 1787828154] req@ffff907d908452c0 x1874670011652352/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828154.101297:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011652352/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828154.101306:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828154.101309:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828154.101312:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011652352:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828154.228752:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828154.228858:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828155.252730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828155.252876:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828156.276735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828156.276753:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828157.300701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828157.300849:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828158.324769:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828158.324810:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828159.220818:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828159.220851:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828159.220859:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828159.220876:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3562 00000100:00080000:3.0:1787828159.220895:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828159.220909:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828159.221007:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011652480/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828159.221074:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828159.221080:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828159.221084:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828159.221098:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3562 00000100:00080000:3.0:1787828159.221107:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828159.221120:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828159.221139:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828159.221155:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011652480:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828159.221168:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011652608/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828159.221195:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828159.221201:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828159.221204:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828159.221210:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3562 00000100:00080000:3.0:1787828159.221214:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828159.221219:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:1.0:1787828159.221320:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:3.0:1787828159.221388:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828159.221400:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828159/real 1787828159] req@ffff907da562d680 x1874670011652480/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828159.221425:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011652480/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828159.221438:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828159.221443:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828159.221448:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011652480:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828159.221477:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011652608:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787828159.221511:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828159.221532:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828159/real 1787828159] req@ffff907da562cf00 x1874670011652608/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828159.221545:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011652608/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828159.221554:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828159.221558:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828159.221562:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011652608:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828159.221748:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011652736/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828159.221787:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828159.221794:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828159.221797:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828159.221807:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3562 00000100:00080000:3.0:1787828159.221814:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828159.221820:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828159.221831:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828159.221855:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011652864/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828159.221873:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011652736:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828159.221945:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828159.221970:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828159/real 1787828159] req@ffff907da562ed00 x1874670011652736/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828159.222004:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011652736/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828159.222019:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828159.222025:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828159.222030:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011652736:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828159.222062:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011652864:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828159.222090:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828159.222104:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828159/real 1787828159] req@ffff907da562c780 x1874670011652864/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828159.222131:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011652864/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828159.222137:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828159.222140:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828159.222143:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011652864:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828159.348738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828159.348873:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828160.372724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828160.372894:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828161.396775:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828161.396925:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828162.420838:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828162.420859:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828163.444815:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828163.444826:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:3.0:1787828164.340721:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828164.340735:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828164.340740:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828164.340753:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3568 00000100:00080000:3.0:1787828164.340768:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828164.340778:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828164.340826:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011652992/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828164.340865:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828164.340870:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828164.340872:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828164.340877:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3568 00000100:00080000:3.0:1787828164.340881:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828164.340885:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828164.340898:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828164.340914:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011653120/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828164.340929:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828164.340933:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828164.340936:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828164.340941:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3568 00000100:00080000:3.0:1787828164.340945:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828164.340949:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828164.340955:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828164.340960:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011652992:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828164.340973:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011653248/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828164.340989:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828164.340995:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828164.340997:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828164.341001:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3568 00000100:00080000:3.0:1787828164.341005:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828164.341008:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828164.341013:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828164.341029:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011653376/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828164.341069:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828164.341123:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828164/real 1787828164] req@ffff907da562f480 x1874670011652992/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828164.341147:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011652992/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828164.341162:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828164.341167:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828164.341173:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011652992:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828164.341183:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011653120:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828164.341212:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828164.341227:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828164/real 1787828164] req@ffff907da562c000 x1874670011653120/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828164.341243:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011653120/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828164.341253:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828164.341257:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828164.341261:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011653120:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828164.341290:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011653248:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828164.341312:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828164.341325:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828164/real 1787828164] req@ffff907da562e580 x1874670011653248/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828164.341368:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011653248/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828164.341380:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828164.341384:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828164.341388:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011653248:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828164.341396:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011653376:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828164.341422:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828164.341435:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828164/real 1787828164] req@ffff907da562d2c0 x1874670011653376/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828164.341449:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011653376/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828164.341459:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828164.341462:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828164.341466:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011653376:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828164.468746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828164.468841:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828165.492807:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828165.492826:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828166.516769:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828166.516956:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828167.540782:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828167.540945:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828168.564717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828168.564873:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828169.460672:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828169.460681:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828169.460685:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828169.460693:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3573 00000100:00080000:3.0:1787828169.460703:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828169.460709:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828169.460741:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011653504/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828169.460763:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828169.460766:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828169.460768:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828169.460777:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3573 00000100:00080000:3.0:1787828169.460780:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828169.460784:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828169.460793:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828169.460804:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011653632/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828169.460813:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011653504:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828169.460821:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828169.460825:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828169.460827:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828169.460831:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3573 00000100:00080000:3.0:1787828169.460834:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828169.460837:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828169.460841:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828169.460852:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011653760/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828169.460859:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828169.460862:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828169.460864:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828169.460867:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3573 00000100:00080000:3.0:1787828169.460869:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828169.460871:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828169.460875:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787828169.460878:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828169.460887:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011653888/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828169.460919:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828169/real 1787828169] req@ffff907da562c3c0 x1874670011653504/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828169.460941:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011653504/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828169.460949:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828169.460953:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828169.460956:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011653504:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828169.460975:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011653632:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828169.460993:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828169.461002:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828169/real 1787828169] req@ffff907da562f0c0 x1874670011653632/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828169.461011:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011653632/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828169.461018:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828169.461020:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828169.461022:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011653632:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828169.461026:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011653760:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828169.461039:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828169.461046:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828169/real 1787828169] req@ffff907da562fc00 x1874670011653760/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828169.461054:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011653760/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828169.461065:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828169.461067:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828169.461070:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011653760:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828169.461073:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011653888:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828169.461085:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828169.461096:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828169/real 1787828169] req@ffff907da562e940 x1874670011653888/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828169.461109:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011653888/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828169.461119:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828169.461122:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828169.461127:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011653888:192.168.203.104@tcp:8:kworker.0 00010000:00010000:0.0:1787828169.588809:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828169.588820:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787828170.612711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828170.612785:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828171.636720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828171.636770:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828172.660706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828172.660814:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828173.684687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828173.684778:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828174.580859:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828174.580872:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828174.580878:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828174.580891:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3578 00000100:00080000:2.0:1787828174.580908:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828174.580917:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828174.580968:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011654016/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828174.581014:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828174.581019:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828174.581021:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828174.581025:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3578 00000100:00080000:2.0:1787828174.581029:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828174.581032:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828174.581045:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828174.581065:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011654144/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828174.581084:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828174.581090:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828174.581095:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828174.581102:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011654016:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828174.581107:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3578 00000100:00080000:2.0:1787828174.581121:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828174.581131:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828174.581143:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828174.581163:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011654272/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828174.581173:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828174.581179:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828174.581183:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828174.581187:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3578 00000100:00080000:2.0:1787828174.581192:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787828174.581200:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787828174.581200:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828174.581214:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828174.581235:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011654400/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828174.581256:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828174/real 1787828174] req@ffff907d9114bc00 x1874670011654016/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828174.581302:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011654016/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828174.581328:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828174.581333:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828174.581338:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011654016:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828174.581363:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011654144:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828174.581390:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828174.581402:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828174/real 1787828174] req@ffff907d911483c0 x1874670011654144/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828174.581413:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011654144/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828174.581421:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828174.581424:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828174.581427:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011654144:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828174.581433:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011654272:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828174.581455:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828174.581470:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828174/real 1787828174] req@ffff907d9114b480 x1874670011654272/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828174.581488:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011654272/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828174.581500:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828174.581505:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828174.581510:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011654272:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828174.581519:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011654400:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828174.581545:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828174.581556:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828174/real 1787828174] req@ffff907d91149680 x1874670011654400/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828174.581567:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011654400/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828174.581575:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828174.581577:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828174.581580:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011654400:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828174.708759:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828174.708897:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828175.732752:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828175.732913:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828176.756780:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828176.756814:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828177.780755:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828177.780858:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828178.804782:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828178.804857:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828179.700725:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828179.700736:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828179.700740:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828179.700750:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3583 00000100:00080000:2.0:1787828179.700761:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828179.700768:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828179.700810:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011654528/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828179.700838:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828179.700841:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828179.700843:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828179.700847:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3583 00000100:00080000:2.0:1787828179.700850:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828179.700853:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828179.700862:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828179.700873:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011654656/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828179.700888:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828179.700892:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828179.700894:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828179.700897:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3583 00000100:00080000:2.0:1787828179.700900:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828179.700905:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828179.700909:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828179.700918:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011654784/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828179.700927:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828179.700931:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828179.700933:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828179.700936:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3583 00000100:00080000:2.0:1787828179.700940:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828179.700943:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828179.700949:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828179.700970:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011654528:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828179.700970:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011654912/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828179.701157:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011654656:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828179.701193:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011654784:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828179.701260:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011654912:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828179.828767:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828179.828839:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828180.852710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828180.852778:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828181.876708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828181.876938:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787828182.772903:0:2338:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:3.0:1787828182.772976:0:2338:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787828182.773000:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787828182.773089:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787828182.773123:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787828182.773155:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:0.0:1787828182.773179:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828179/real 1787828182] req@ffff907d91149680 x1874670011654528/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828182.773259:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011654528/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828182.773281:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828182.773290:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828182.773298:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011654528:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828182.773313:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828179/real 1787828182] req@ffff907d9114b480 x1874670011654656/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828182.773339:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011654656/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828182.773353:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828182.773359:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828182.773365:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011654656:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828182.773377:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828179/real 1787828182] req@ffff907d911483c0 x1874670011654784/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828182.773402:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011654784/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828182.773418:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828182.773423:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828182.773429:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011654784:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828182.773464:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828179/real 1787828182] req@ffff907d9114bc00 x1874670011654912/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828182.773488:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011654912/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828182.773503:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828182.773509:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828182.773515:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011654912:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787828182.900746:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828182.900823:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787828183.924705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828183.924759:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828184.820783:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828184.820824:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828184.820831:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828184.820845:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3588 00000100:00080000:1.0:1787828184.820862:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828184.820873:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828184.820919:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011655040/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828184.820970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828184.820977:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828184.820981:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828184.820988:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3588 00000100:00080000:1.0:1787828184.820995:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828184.821002:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828184.821016:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828184.821052:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011655168/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828184.821072:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828184.821079:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828184.821083:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828184.821091:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3588 00000100:00080000:1.0:1787828184.821099:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828184.821107:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828184.821117:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828184.821161:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011655040:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828184.821172:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011655296/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828184.821253:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828184.821259:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828184.821262:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828184.821271:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3588 00000100:00080000:1.0:1787828184.821278:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828184.821286:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828184.821296:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828184.821323:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011655424/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828184.821361:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828184.821435:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828184/real 1787828184] req@ffff907d908a21c0 x1874670011655040/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828184.821476:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011655040/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828184.821490:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828184.821496:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828184.821500:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011655040:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828184.821511:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011655168:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828184.821557:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828184.821574:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828184/real 1787828184] req@ffff907d908a2580 x1874670011655168/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828184.821589:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011655168/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828184.821600:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828184.821604:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828184.821624:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011655168:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828184.821694:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011655296:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828184.821723:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828184.821737:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828184/real 1787828184] req@ffff907d908a2940 x1874670011655296/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828184.821750:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011655296/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828184.821759:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828184.821763:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828184.821767:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011655296:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828184.821774:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011655424:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828184.821797:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828184.821809:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828184/real 1787828184] req@ffff907d908a0000 x1874670011655424/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828184.821822:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011655424/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828184.821833:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828184.821837:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828184.821841:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011655424:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828184.948757:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828184.948923:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828185.972694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828185.972739:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828186.996710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828186.996755:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828188.020690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828188.020742:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828189.044748:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828189.044872:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828189.940717:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828189.940728:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828189.940733:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828189.940744:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3593 00000100:00080000:1.0:1787828189.940757:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828189.940767:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828189.940808:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011655552/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828189.940849:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828189.940853:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828189.940855:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828189.940866:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3593 00000100:00080000:1.0:1787828189.940870:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828189.940874:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828189.940885:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828189.940916:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011655680/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828189.940929:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828189.940934:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828189.940948:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828189.940949:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011655552:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828189.940958:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3593 00000100:00080000:1.0:1787828189.940981:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828189.940985:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828189.940991:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828189.941008:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011655808/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828189.941021:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828189.941025:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828189.941027:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828189.941031:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3593 00000100:00080000:1.0:1787828189.941037:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828189.941041:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828189.941047:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787828189.941052:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828189.941086:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011655936/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828189.941132:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828189/real 1787828189] req@ffff907d908a1e00 x1874670011655552/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828189.941187:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011655552/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828189.941224:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828189.941228:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828189.941233:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011655552:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828189.941241:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011655680:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828189.941277:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828189.941291:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828189/real 1787828189] req@ffff907d908a2d00 x1874670011655680/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828189.941303:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011655680/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828189.941312:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828189.941315:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828189.941318:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011655680:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828189.941344:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011655808:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828189.941362:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828189.941390:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828189/real 1787828189] req@ffff907d908a0000 x1874670011655808/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828189.941405:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011655808/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828189.941416:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828189.941420:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828189.941424:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011655808:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828189.941442:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011655936:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828189.941463:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828189.941474:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828189/real 1787828189] req@ffff907d908a2940 x1874670011655936/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828189.941488:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011655936/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828189.941498:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828189.941501:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828189.941505:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011655936:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828190.068701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828190.068748:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828191.092728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828191.092766:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828192.116730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828192.116762:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828193.140688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828193.140774:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828194.164718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828194.164741:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828195.060719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828195.060731:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828195.060737:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828195.060750:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3598 00000100:00080000:3.0:1787828195.060762:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828195.060771:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828195.060811:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011656064/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828195.060846:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828195.060852:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828195.060855:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828195.060862:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3598 00000100:00080000:3.0:1787828195.060868:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828195.060875:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828195.060888:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828195.060891:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011656064:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828195.060909:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011656192/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828195.060924:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828195.060928:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828195.060931:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828195.060937:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3598 00000100:00080000:3.0:1787828195.060943:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828195.060949:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828195.060955:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828195.060972:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011656320/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828195.060982:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787828195.060987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828195.060992:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828195.060995:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828195.061000:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3598 00000100:00080000:3.0:1787828195.061004:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828195.061010:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828195.061017:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828195.061034:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011656448/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828195.061058:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828195/real 1787828195] req@ffff907da562da40 x1874670011656064/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828195.061084:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011656064/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828195.061098:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828195.061104:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828195.061111:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011656064:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828195.061164:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011656192:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787828195.061199:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828195.061219:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828195/real 1787828195] req@ffff907da562f840 x1874670011656192/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828195.061244:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011656192/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828195.061255:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828195.061259:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828195.061264:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011656192:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828195.061273:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011656320:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828195.061304:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828195.061321:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828195/real 1787828195] req@ffff907da562e1c0 x1874670011656320/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828195.061339:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011656320/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828195.061350:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828195.061355:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828195.061364:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011656320:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828195.061376:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011656448:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828195.061405:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828195.061420:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828195/real 1787828195] req@ffff907da562cb40 x1874670011656448/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828195.061438:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011656448/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828195.061459:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828195.061465:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828195.061468:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011656448:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828195.188682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828195.188760:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828196.212676:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828196.212721:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828197.236671:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828197.236745:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828198.260697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828198.260701:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828199.284736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828199.284825:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828200.180687:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828200.180698:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828200.180702:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828200.180712:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3603 00000100:00080000:3.0:1787828200.180722:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828200.180729:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828200.180764:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011656576/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828200.180790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828200.180795:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828200.180797:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828200.180801:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3603 00000100:00080000:3.0:1787828200.180806:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828200.180811:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828200.180821:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828200.180824:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011656576:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828200.180839:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011656704/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828200.180853:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828200.180857:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828200.180860:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828200.180864:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3603 00000100:00080000:3.0:1787828200.180868:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828200.180873:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828200.180879:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787828200.180880:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828200.180897:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011656832/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828200.180910:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828200.180913:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828200.180915:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828200.180918:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828200/real 1787828200] req@ffff907da562de00 x1874670011656576/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828200.180922:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3603 00000100:00080000:3.0:1787828200.180930:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828200.180936:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828200.180940:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011656576/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787828200.180944:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1787828200.180951:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828200.180955:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828200.180959:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011656576:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828200.180964:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011656960/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828200.180981:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011656704:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787828200.180999:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828200.181008:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828200/real 1787828200] req@ffff907da562cb40 x1874670011656704/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828200.181016:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011656704/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828200.181021:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828200.181024:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828200.181026:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011656704:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828200.181029:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011656832:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828200.181040:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828200.181045:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828200/real 1787828200] req@ffff907da562e1c0 x1874670011656832/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828200.181052:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011656832/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828200.181058:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828200.181059:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828200.181061:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011656832:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828200.181066:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011656960:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828200.181096:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828200.181108:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828200/real 1787828200] req@ffff907da562f840 x1874670011656960/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828200.181121:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011656960/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828200.181130:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828200.181133:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828200.181137:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011656960:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828200.308681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828200.308698:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828201.332690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828201.332710:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828202.356691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828202.356726:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828203.380704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828203.380766:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828204.404683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828204.404751:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828205.300772:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828205.300786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828205.300792:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828205.300806:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3609 00000100:00080000:3.0:1787828205.300820:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828205.300830:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828205.300891:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011657088/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828205.300930:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828205.300936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828205.300938:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828205.300943:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3609 00000100:00080000:3.0:1787828205.300948:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828205.300954:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828205.300969:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828205.300985:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011657088:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828205.300988:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011657216/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828205.301004:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828205.301009:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828205.301011:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828205.301015:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3609 00000100:00080000:3.0:1787828205.301020:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828205.301025:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828205.301032:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828205.301060:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011657344/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828205.301072:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787828205.301077:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787828205.301081:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828205.301085:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828205.301091:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3609 00000100:00080000:3.0:1787828205.301096:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828205.301101:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828205.301108:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828205.301126:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011657472/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828205.301131:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828205/real 1787828205] req@ffff907da562da40 x1874670011657088/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828205.301161:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011657088/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828205.301171:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828205.301175:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828205.301179:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011657088:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828205.301202:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011657216:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787828205.301220:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828205.301229:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828205/real 1787828205] req@ffff907da562e940 x1874670011657216/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828205.301238:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011657216/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828205.301244:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828205.301246:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828205.301249:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011657216:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828205.301254:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011657344:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828205.301274:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828205.301281:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828205/real 1787828205] req@ffff907da562fc00 x1874670011657344/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828205.301289:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011657344/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828205.301294:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828205.301296:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828205.301299:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011657344:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828205.301302:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011657472:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828205.301312:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828205.301318:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828205/real 1787828205] req@ffff907da562f0c0 x1874670011657472/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828205.301326:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011657472/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828205.301331:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828205.301332:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828205.301334:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011657472:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828205.428720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828205.428814:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828206.452712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828206.452770:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828207.476729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828207.476768:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828208.500711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828208.500730:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828209.524692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828209.524743:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828210.420702:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828210.420710:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828210.420714:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828210.420722:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3614 00000100:00080000:1.0:1787828210.420732:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828210.420740:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828210.420778:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011657600/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828210.420806:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828210.420808:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828210.420810:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828210.420813:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3614 00000100:00080000:1.0:1787828210.420816:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828210.420819:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828210.420827:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828210.420836:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011657728/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828210.420843:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828210.420845:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828210.420846:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828210.420849:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3614 00000100:00080000:1.0:1787828210.420851:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828210.420854:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828210.420857:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828210.420864:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011657856/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828210.420871:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828210.420874:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828210.420875:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828210.420877:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3614 00000100:00080000:1.0:1787828210.420879:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828210.420883:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011657600:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828210.420884:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828210.420891:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828210.420897:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011657984/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828210.420944:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828210.420979:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828210/real 1787828210] req@ffff907d908a2580 x1874670011657600/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828210.420992:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011657600/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828210.421000:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828210.421004:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828210.421023:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011657600:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828210.421031:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011657728:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828210.421057:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828210.421070:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828210/real 1787828210] req@ffff907d908a21c0 x1874670011657728/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828210.421084:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011657728/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828210.421092:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828210.421096:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828210.421099:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011657728:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828210.421106:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011657856:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828210.421126:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828210.421135:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828210/real 1787828210] req@ffff907d908a0f00 x1874670011657856/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828210.421143:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011657856/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828210.421149:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828210.421151:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828210.421153:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011657856:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828210.421172:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011657984:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828210.421184:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828210.421191:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828210/real 1787828210] req@ffff907d908a12c0 x1874670011657984/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828210.421211:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011657984/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828210.421217:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828210.421219:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828210.421221:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011657984:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828210.548692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828210.548762:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828211.572655:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828211.572681:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828212.596684:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828212.596709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787828213.620712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828213.620788:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828214.644674:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828214.644716:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828215.540759:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828215.540771:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828215.540776:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828215.540787:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3619 00000100:00080000:3.0:1787828215.540800:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828215.540807:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828215.540849:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011658112/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828215.540880:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828215.540883:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828215.540885:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828215.540888:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3619 00000100:00080000:3.0:1787828215.540890:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828215.540894:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828215.540903:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828215.540913:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011658240/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828215.540921:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828215.540924:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828215.540925:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828215.540928:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3619 00000100:00080000:3.0:1787828215.540931:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828215.540936:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011658112:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828215.540937:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828215.540945:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828215.540952:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011658368/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828215.540958:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828215.540961:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828215.540963:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828215.540965:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3619 00000100:00080000:3.0:1787828215.540967:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828215.540969:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828215.540985:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828215.540997:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011658496/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828215.541026:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828215.541069:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828215/real 1787828215] req@ffff907da562c3c0 x1874670011658112/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828215.541085:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011658112/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828215.541094:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828215.541097:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828215.541101:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011658112:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828215.541107:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011658240:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828215.541123:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828215.541132:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828215/real 1787828215] req@ffff907da562d2c0 x1874670011658240/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828215.541140:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011658240/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828215.541151:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828215.541153:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828215.541155:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011658240:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828215.541178:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011658368:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828215.541191:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828215.541198:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828215/real 1787828215] req@ffff907da562e580 x1874670011658368/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828215.541205:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011658368/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828215.541211:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828215.541213:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828215.541215:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011658368:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828215.541219:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011658496:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828215.541229:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828215.541235:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828215/real 1787828215] req@ffff907da562c000 x1874670011658496/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828215.541242:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011658496/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828215.541247:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828215.541249:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828215.541252:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011658496:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828215.668734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828215.668762:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828216.692706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828216.692790:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828217.716686:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828217.716735:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828218.740695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828218.740748:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828219.764743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828219.764816:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828220.660762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828220.660770:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828220.660773:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828220.660798:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3624 00000100:00080000:2.0:1787828220.660809:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828220.660816:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828220.660870:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011658624/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828220.660892:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828220.660895:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828220.660896:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828220.660899:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3624 00000100:00080000:2.0:1787828220.660902:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828220.660904:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828220.660911:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828220.660922:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011658752/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828220.660930:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828220.660932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828220.660933:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828220.660953:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011658624:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828220.660955:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3624 00000100:00080000:2.0:1787828220.660961:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828220.660966:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828220.660970:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828220.660981:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011658880/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828220.660987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828220.660990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828220.660992:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828220.660994:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3624 00000100:00080000:2.0:1787828220.660998:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828220.661001:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828220.661004:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828220.661010:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011659008/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828220.661028:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828220.661064:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828220/real 1787828220] req@ffff907d9114b840 x1874670011658624/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828220.661078:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011658624/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828220.661086:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828220.661089:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828220.661091:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011658624:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828220.661108:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011658752:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828220.661126:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828220.661140:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828220/real 1787828220] req@ffff907d91149a40 x1874670011658752/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828220.661153:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011658752/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828220.661163:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828220.661166:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828220.661170:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011658752:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828220.661176:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011658880:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828220.661195:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828220.661203:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828220/real 1787828220] req@ffff907d9114ad00 x1874670011658880/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828220.661211:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011658880/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828220.661216:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828220.661219:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828220.661221:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011658880:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828220.661224:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011659008:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828220.661235:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828220.661240:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828220/real 1787828220] req@ffff907d9114a580 x1874670011659008/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828220.661247:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011659008/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828220.661253:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828220.661254:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828220.661256:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011659008:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828220.788715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828220.788803:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828221.812724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828221.812744:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828222.836765:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828222.836820:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828223.860734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828223.860790:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828224.884749:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828224.884751:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:3.0:1787828225.780719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828225.780730:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828225.780735:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828225.780750:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3629 00000100:00080000:3.0:1787828225.780763:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828225.780770:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828225.780814:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011659136/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828225.780840:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828225.780843:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828225.780844:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828225.780847:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3629 00000100:00080000:3.0:1787828225.780850:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828225.780852:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828225.780862:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828225.780871:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011659264/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828225.780879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828225.780881:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828225.780888:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828225.780893:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3629 00000100:00080000:3.0:1787828225.780895:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828225.780899:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828225.780916:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828225.780927:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011659392/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828225.780935:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828225.780938:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828225.780941:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828225.780947:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3629 00000100:00100000:0.0:1787828225.780950:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011659136:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828225.780950:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828225.780958:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828225.780967:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828225.780977:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011659520/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828225.781042:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828225.781092:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828225/real 1787828225] req@ffff907da562f480 x1874670011659136/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828225.781113:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011659136/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828225.781135:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828225.781140:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828225.781144:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011659136:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828225.781154:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011659264:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828225.781179:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828225.781194:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828225/real 1787828225] req@ffff907da562c780 x1874670011659264/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828225.781208:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011659264/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828225.781217:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828225.781221:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828225.781225:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011659264:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828225.781253:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011659392:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828225.781277:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828225.781290:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828225/real 1787828225] req@ffff907da562ed00 x1874670011659392/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828225.781303:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011659392/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828225.781318:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828225.781322:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828225.781326:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011659392:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828225.781333:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011659520:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828225.781352:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828225.781363:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828225/real 1787828225] req@ffff907da562cf00 x1874670011659520/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828225.781375:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011659520/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828225.781384:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828225.781387:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828225.781390:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011659520:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828225.908727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828225.908757:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828226.932695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828226.932755:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828227.956724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828227.956796:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828228.980661:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828228.980737:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828230.004698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828230.004730:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828230.900693:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828230.900702:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828230.900706:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828230.900714:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3634 00000100:00080000:3.0:1787828230.900723:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828230.900730:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828230.900761:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011659648/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828230.900788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828230.900790:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828230.900792:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828230.900795:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3634 00000100:00080000:3.0:1787828230.900798:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828230.900801:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828230.900809:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828230.900820:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011659776/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828230.900828:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828230.900832:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011659648:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828230.900832:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828230.900835:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828230.900841:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3634 00000100:00080000:3.0:1787828230.900845:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828230.900848:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828230.900852:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828230.900862:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011659904/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828230.900882:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828230.900885:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828230.900887:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828230.900891:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3634 00000100:00080000:3.0:1787828230.900902:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828230.900906:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828230.900910:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787828230.900916:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828230.900924:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011660032/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828230.900957:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828230/real 1787828230] req@ffff907da562d680 x1874670011659648/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828230.900977:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011659648/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828230.900992:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828230.900997:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828230.901003:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011659648:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828230.901029:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011659776:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828230.901053:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828230.901066:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828230/real 1787828230] req@ffff907da562cf00 x1874670011659776/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828230.901079:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011659776/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828230.901088:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828230.901092:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828230.901096:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011659776:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828230.901104:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011659904:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828230.901125:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828230.901139:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828230/real 1787828230] req@ffff907da562ed00 x1874670011659904/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828230.901152:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011659904/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828230.901163:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828230.901167:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828230.901172:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011659904:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828230.901180:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011660032:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828230.901198:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828230.901213:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828230/real 1787828230] req@ffff907da562c780 x1874670011660032/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828230.901225:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011660032/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828230.901234:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828230.901237:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828230.901240:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011660032:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828231.028676:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828231.028757:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828232.052717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828232.052779:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828233.076664:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828233.076727:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828234.100716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828234.100814:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828235.124817:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828235.124956:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828236.020736:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828236.020753:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828236.020760:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828236.020773:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3639 00000100:00080000:3.0:1787828236.020787:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828236.020796:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828236.020875:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011660160/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828236.020928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828236.020933:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828236.020936:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828236.020941:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3639 00000100:00080000:3.0:1787828236.020945:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828236.020950:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828236.020964:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828236.020981:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011660288/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828236.020995:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828236.021000:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828236.021003:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828236.021008:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3639 00000100:00080000:3.0:1787828236.021011:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828236.021019:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011660160:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828236.021022:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828236.021036:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828236.021054:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011660416/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828236.021068:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828236.021073:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828236.021076:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828236.021081:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3639 00000100:00080000:3.0:1787828236.021102:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828236.021107:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828236.021114:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787828236.021115:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828236.021137:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011660544/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828236.021181:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828236/real 1787828236] req@ffff907da562f480 x1874670011660160/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828236.021208:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011660160/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828236.021222:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828236.021227:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828236.021233:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011660160:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828236.021243:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011660288:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828236.021270:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828236.021295:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828236/real 1787828236] req@ffff907da562c000 x1874670011660288/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828236.021310:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011660288/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828236.021320:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828236.021324:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828236.021328:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011660288:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828236.021359:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011660416:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828236.021380:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828236.021392:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828236/real 1787828236] req@ffff907da562e580 x1874670011660416/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828236.021407:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011660416/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828236.021417:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828236.021421:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828236.021425:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011660416:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828236.021431:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011660544:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828236.021449:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828236.021460:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828236/real 1787828236] req@ffff907da562d2c0 x1874670011660544/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828236.021474:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011660544/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828236.021484:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828236.021487:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828236.021491:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011660544:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828236.148765:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828236.148894:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828237.172738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828237.172846:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828238.196761:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828238.196900:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828239.220706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828239.220762:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828240.244784:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828240.244841:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828241.140687:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828241.140699:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828241.140703:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828241.140713:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3644 00000100:00080000:3.0:1787828241.140725:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828241.140733:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828241.140772:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011660672/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828241.140815:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828241.140819:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828241.140821:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828241.140826:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3644 00000100:00080000:3.0:1787828241.140830:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828241.140835:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828241.140845:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828241.140856:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011660672:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828241.140859:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011660800/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828241.140869:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828241.140873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828241.140874:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828241.140878:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3644 00000100:00080000:3.0:1787828241.140881:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828241.140884:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828241.140889:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828241.140898:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011660928/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828241.140906:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828241.140909:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828241.140911:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828241.140914:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3644 00000800:00000100:2.0:1787828241.140916:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787828241.140918:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828241.140921:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828241.140936:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828241.140947:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011661056/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828241.140975:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828241/real 1787828241] req@ffff907da562c3c0 x1874670011660672/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828241.140995:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011660672/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828241.141003:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828241.141006:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828241.141009:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011660672:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828241.141029:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011660800:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828241.141048:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828241.141057:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828241/real 1787828241] req@ffff907da562f0c0 x1874670011660800/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828241.141065:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011660800/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828241.141071:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828241.141075:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828241.141078:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011660800:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828241.141084:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011660928:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828241.141105:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828241.141117:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828241/real 1787828241] req@ffff907da562fc00 x1874670011660928/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828241.141128:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011660928/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828241.141143:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828241.141146:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828241.141150:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011660928:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828241.141156:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011661056:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828241.141175:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828241.141186:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828241/real 1787828241] req@ffff907da562e940 x1874670011661056/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828241.141196:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011661056/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828241.141204:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828241.141208:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828241.141211:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011661056:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828241.268762:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828241.268785:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828242.292723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828242.292903:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828243.316714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828243.316872:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828244.340730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828244.340768:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828245.364860:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828245.364878:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:3.0:1787828246.260688:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828246.260701:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828246.260706:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828246.260718:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3650 00000100:00080000:3.0:1787828246.260731:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828246.260740:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828246.260826:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011661184/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828246.260853:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828246.260857:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828246.260860:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828246.260864:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3650 00000100:00080000:3.0:1787828246.260868:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828246.260871:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828246.260881:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828246.260895:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011661312/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828246.260906:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828246.260909:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828246.260911:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828246.260914:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3650 00000100:00080000:3.0:1787828246.260918:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828246.260925:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828246.260926:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011661184:192.168.203.104@tcp:250:kworker.0 00000008:00100000:3.0:1787828246.260937:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828246.260948:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011661440/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828246.260958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828246.260962:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828246.260964:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828246.260968:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3650 00000100:00080000:3.0:1787828246.260972:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828246.260976:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828246.260982:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828246.260992:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011661568/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828246.261074:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011661312:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828246.261148:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011661440:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828246.261186:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011661568:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828246.388714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828246.388817:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828247.412820:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828247.412908:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828248.436722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828248.436802:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787828249.332890:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:0.0:1787828249.332932:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787828249.332950:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787828249.332986:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787828249.333006:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787828249.333015:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:2.0:1787828249.333022:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828246/real 1787828249] req@ffff907da562da40 x1874670011661184/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828249.333043:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011661184/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828249.333052:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828249.333055:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828249.333058:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011661184:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828249.333064:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828246/real 1787828249] req@ffff907da562f840 x1874670011661312/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828249.333071:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011661312/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828249.333076:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828249.333079:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828249.333081:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011661312:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828249.333104:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828246/real 1787828249] req@ffff907da562e1c0 x1874670011661440/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828249.333116:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011661440/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828249.333125:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828249.333129:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828249.333132:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011661440:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828249.333138:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828246/real 1787828249] req@ffff907da562cb40 x1874670011661568/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828249.333149:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011661568/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828249.333156:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828249.333159:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828249.333162:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011661568:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828249.460730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828249.460805:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828250.484681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828250.484728:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828251.380659:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828251.380668:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828251.380671:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828251.380680:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3655 00000100:00080000:3.0:1787828251.380689:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828251.380696:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828251.380740:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011661696/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828251.380762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828251.380765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828251.380767:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828251.380769:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3655 00000100:00080000:3.0:1787828251.380772:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828251.380775:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828251.380783:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828251.380796:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011661824/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828251.380805:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011661696:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828251.380807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828251.380814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828251.380816:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828251.380819:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3655 00000100:00080000:3.0:1787828251.380821:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828251.380824:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828251.380827:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828251.380836:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011661952/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828251.380843:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828251.380846:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828251.380847:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828251.380849:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3655 00000100:00080000:3.0:1787828251.380854:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828251.380857:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828251.380859:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787828251.380864:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828251.380869:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011662080/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828251.380899:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828251/real 1787828251] req@ffff907da562de00 x1874670011661696/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828251.380912:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011661696/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828251.380920:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828251.380923:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828251.380926:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011661696:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828251.380944:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011661824:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828251.380958:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828251.380967:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828251/real 1787828251] req@ffff907da562cb40 x1874670011661824/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828251.380975:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011661824/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828251.380980:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828251.380982:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828251.380985:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011661824:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828251.380988:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011661952:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828251.380999:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828251.381005:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828251/real 1787828251] req@ffff907da562e1c0 x1874670011661952/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828251.381012:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011661952/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828251.381028:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828251.381030:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828251.381032:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011661952:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828251.381036:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011662080:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828251.381047:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828251.381053:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828251/real 1787828251] req@ffff907da562f840 x1874670011662080/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828251.381060:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011662080/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828251.381066:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828251.381068:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828251.381070:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011662080:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828251.508717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828251.508822:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828252.532679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828252.532730:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828253.556723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828253.556763:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828254.580777:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828254.580833:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828255.604720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828255.604770:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828256.500828:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828256.500842:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828256.500847:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828256.500860:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3660 00000100:00080000:3.0:1787828256.500885:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828256.500894:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828256.500950:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011662208/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828256.500990:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828256.500995:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828256.500998:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828256.501003:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3660 00000100:00080000:3.0:1787828256.501008:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828256.501014:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828256.501026:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828256.501055:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011662336/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828256.501069:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011662208:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828256.501076:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828256.501083:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828256.501086:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828256.501093:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3660 00000100:00080000:3.0:1787828256.501098:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828256.501104:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828256.501112:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828256.501130:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011662464/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828256.501141:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787828256.501162:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828256.501167:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828256.501169:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828256.501175:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3660 00000100:00080000:3.0:1787828256.501181:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828256.501189:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828256/real 1787828256] req@ffff907da562da40 x1874670011662208/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828256.501190:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828256.501203:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828256.501214:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011662208/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828256.501226:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011662592/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828256.501228:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828256.501237:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828256.501254:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011662208:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828256.501282:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011662336:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828256.501303:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828256.501322:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828256/real 1787828256] req@ffff907da562e940 x1874670011662336/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828256.501332:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011662336/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828256.501339:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828256.501342:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828256.501345:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011662336:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828256.501349:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011662464:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828256.501363:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828256.501370:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828256/real 1787828256] req@ffff907da562fc00 x1874670011662464/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828256.501380:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011662464/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828256.501387:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828256.501389:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828256.501391:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011662464:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828256.501396:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011662592:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828256.501407:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828256.501415:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828256/real 1787828256] req@ffff907da562f0c0 x1874670011662592/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828256.501431:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011662592/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828256.501441:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828256.501445:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828256.501450:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011662592:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828256.628707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828256.628794:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828257.652790:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828257.652879:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828258.676695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828258.676754:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828259.700702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828259.700741:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828260.724734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828260.724827:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828261.620762:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828261.620781:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828261.620789:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828261.620808:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3665 00000100:00080000:3.0:1787828261.620829:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828261.620843:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828261.620910:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011662720/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828261.620964:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828261.620971:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828261.620977:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828261.620987:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3665 00000100:00080000:3.0:1787828261.620997:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828261.621003:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828261.621019:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828261.621041:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011662720:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828261.621043:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011662848/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828261.621081:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828261.621089:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828261.621093:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828261.621103:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3665 00000100:00080000:3.0:1787828261.621128:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787828261.621135:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787828261.621145:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828261.621157:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828261.621178:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828261/real 1787828261] req@ffff907da562c3c0 x1874670011662720/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828261.621185:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011662976/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828261.621239:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828261.621240:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011662720/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828261.621258:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828261.621267:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828261.621273:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828261.621282:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3665 00000100:00080000:2.0:1787828261.621289:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00080000:3.0:1787828261.621296:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828261.621298:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011662720:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828261.621306:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828261.621315:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828261.621329:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011662848:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828261.621337:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011663104/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828261.621358:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828261.621374:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828261/real 1787828261] req@ffff907da562d2c0 x1874670011662848/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828261.621386:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011662848/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828261.621392:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828261.621395:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828261.621398:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011662848:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828261.621403:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011662976:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828261.621415:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828261.621421:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828261/real 1787828261] req@ffff907da562e580 x1874670011662976/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828261.621428:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011662976/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828261.621434:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828261.621436:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828261.621438:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011662976:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828261.621456:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011663104:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828261.621468:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828261.621475:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828261/real 1787828261] req@ffff907da562c000 x1874670011663104/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828261.621482:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011663104/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828261.621488:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828261.621491:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828261.621495:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011663104:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828261.748769:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828261.748849:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828262.772728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828262.772829:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828263.796762:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828263.796879:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828264.820788:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828264.820926:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828265.844759:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828265.844878:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828266.740725:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828266.740735:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828266.740740:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828266.740750:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3670 00000100:00080000:3.0:1787828266.740759:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828266.740766:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828266.740800:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011663232/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828266.740825:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828266.740828:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828266.740829:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828266.740833:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3670 00000100:00080000:3.0:1787828266.740835:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828266.740838:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828266.740846:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828266.740858:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011663360/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828266.740868:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828266.740872:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828266.740874:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828266.740879:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011663232:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828266.740880:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3670 00000100:00080000:3.0:1787828266.740887:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828266.740893:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828266.740897:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828266.740906:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011663488/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828266.740914:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828266.740917:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828266.740918:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828266.740921:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3670 00000100:00080000:3.0:1787828266.740924:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828266.740927:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828266.740930:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828266.740938:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011663616/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828266.740961:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828266.741004:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828266/real 1787828266] req@ffff907da562f480 x1874670011663232/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828266.741022:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011663232/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828266.741032:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828266.741036:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828266.741041:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011663232:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828266.741066:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011663360:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828266.741102:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828266.741121:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828266/real 1787828266] req@ffff907da562c780 x1874670011663360/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828266.741138:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011663360/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828266.741146:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828266.741150:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828266.741153:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011663360:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828266.741160:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011663488:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828266.741187:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828266.741199:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828266/real 1787828266] req@ffff907da562ed00 x1874670011663488/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828266.741209:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011663488/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828266.741216:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828266.741218:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828266.741222:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011663488:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828266.741226:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011663616:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828266.741241:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828266.741249:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828266/real 1787828266] req@ffff907da562cf00 x1874670011663616/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828266.741259:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011663616/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828266.741267:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828266.741270:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828266.741273:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011663616:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828266.868699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828266.868761:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828267.892697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828267.892751:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828268.916703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828268.916792:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828269.940867:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828269.940914:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828270.964800:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828270.964870:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828271.860795:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828271.860807:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828271.860812:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828271.860843:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3675 00000100:00080000:1.0:1787828271.860857:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828271.860868:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828271.860916:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011663744/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828271.860949:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828271.860954:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828271.860957:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828271.860962:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3675 00000100:00080000:1.0:1787828271.860966:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828271.860970:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828271.860982:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828271.860998:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011663872/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828271.861031:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828271.861035:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828271.861038:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828271.861047:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011663744:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828271.861049:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3675 00000100:00080000:1.0:1787828271.861058:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828271.861065:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828271.861072:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828271.861090:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011664000/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828271.861102:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828271.861107:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828271.861110:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828271.861115:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3675 00000100:00080000:1.0:1787828271.861119:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828271.861124:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828271.861130:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787828271.861140:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828271.861147:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011664128/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828271.861199:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828271/real 1787828271] req@ffff907d908a1680 x1874670011663744/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828271.861238:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011663744/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828271.861251:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828271.861256:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828271.861261:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011663744:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828271.861289:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011663872:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828271.861316:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828271.861331:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828271/real 1787828271] req@ffff907d908a3840 x1874670011663872/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828271.861362:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011663872/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828271.861373:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828271.861377:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828271.861381:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011663872:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828271.861388:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011664000:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828271.861428:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828271.861440:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828271/real 1787828271] req@ffff907d908a12c0 x1874670011664000/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828271.861461:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011664000/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828271.861470:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828271.861473:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828271.861477:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011664000:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828271.861483:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011664128:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828271.861518:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828271.861530:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828271/real 1787828271] req@ffff907d908a0f00 x1874670011664128/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828271.861544:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011664128/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828271.861554:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828271.861558:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828271.861561:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011664128:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828271.988768:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828271.988826:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828273.012763:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828273.012871:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828274.036699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828274.036753:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828275.060730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828275.060823:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828276.084688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828276.084735:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828276.980679:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828276.980688:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828276.980691:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828276.980700:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3680 00000100:00080000:1.0:1787828276.980708:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828276.980714:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828276.980753:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011664256/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828276.980773:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828276.980776:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828276.980777:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828276.980780:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3680 00000100:00080000:1.0:1787828276.980782:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828276.980784:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828276.980792:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828276.980801:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011664384/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828276.980808:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828276.980811:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828276.980812:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828276.980815:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3680 00000100:00080000:1.0:1787828276.980818:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828276.980823:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011664256:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828276.980824:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828276.980829:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828276.980837:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011664512/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828276.980844:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828276.980847:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828276.980848:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828276.980850:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3680 00000100:00080000:1.0:1787828276.980852:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828276.980854:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828276.980857:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828276.980866:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011664640/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828276.980888:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828276.980928:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828276/real 1787828276] req@ffff907d908a21c0 x1874670011664256/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828276.980948:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011664256/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828276.980960:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828276.980964:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828276.980968:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011664256:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828276.980977:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011664384:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828276.980999:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828276.981011:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828276/real 1787828276] req@ffff907d908a2580 x1874670011664384/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828276.981025:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011664384/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828276.981034:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828276.981038:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828276.981042:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011664384:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828276.981064:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011664512:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828276.981083:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828276.981094:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828276/real 1787828276] req@ffff907d908a2940 x1874670011664512/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828276.981107:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011664512/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828276.981124:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828276.981127:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828276.981131:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011664512:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828276.981137:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011664640:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828276.981155:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828276.981166:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828276/real 1787828276] req@ffff907d908a0000 x1874670011664640/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828276.981178:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011664640/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828276.981187:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828276.981190:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828276.981194:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011664640:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828277.108707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828277.108765:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828278.132666:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828278.132739:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828279.156681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828279.156727:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828280.180674:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828280.180731:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828281.204686:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828281.204729:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828282.101666:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828282.101677:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828282.101682:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828282.101691:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3685 00000100:00080000:1.0:1787828282.101703:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828282.101710:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828282.101754:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011664768/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828282.101781:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828282.101785:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828282.101790:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828282.101795:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3685 00000100:00080000:1.0:1787828282.101799:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828282.101803:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828282.101813:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828282.101828:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011664896/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828282.101836:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011664768:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828282.101844:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828282.101849:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828282.101851:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828282.101855:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3685 00000100:00080000:1.0:1787828282.101860:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828282.101864:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828282.101870:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828282.101884:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011665024/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828282.101895:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828282.101898:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828282.101899:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828282.101904:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3685 00000800:00000100:2.0:1787828282.101907:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787828282.101909:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828282.101914:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828282.101918:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828282.101927:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011665152/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828282.101950:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828282/real 1787828282] req@ffff907d908a2d00 x1874670011664768/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828282.101968:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011664768/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828282.101979:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828282.101982:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828282.101986:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011664768:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828282.102008:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011664896:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828282.102032:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828282.102046:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828282/real 1787828282] req@ffff907d908a1e00 x1874670011664896/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828282.102060:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011664896/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828282.102069:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828282.102073:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828282.102077:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011664896:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828282.102084:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011665024:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828282.102105:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828282.102117:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828282/real 1787828282] req@ffff907d908a0000 x1874670011665024/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828282.102131:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011665024/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828282.102150:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828282.102154:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828282.102158:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011665024:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828282.102165:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011665152:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828282.102185:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828282.102198:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828282/real 1787828282] req@ffff907d908a2940 x1874670011665152/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828282.102211:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011665152/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828282.102220:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828282.102223:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828282.102227:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011665152:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828282.228683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828282.228732:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828283.252704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828283.252784:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828284.276677:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828284.276738:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828285.300684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828285.300743:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828286.324702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828286.324795:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828287.220678:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828287.220686:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828287.220689:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828287.220698:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3690 00000100:00080000:1.0:1787828287.220708:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828287.220714:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828287.220743:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011665280/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828287.220766:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828287.220768:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828287.220770:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828287.220772:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3690 00000100:00080000:1.0:1787828287.220774:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828287.220777:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828287.220784:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828287.220793:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011665408/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828287.220801:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828287.220808:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1787828287.220809:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011665280:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828287.220810:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828287.220817:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3690 00000100:00080000:1.0:1787828287.220820:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828287.220823:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828287.220826:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828287.220835:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011665536/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828287.220842:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828287.220845:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828287.220846:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828287.220848:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3690 00000100:00080000:1.0:1787828287.220852:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828287.220854:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828287.220857:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828287.220863:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011665664/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828287.220873:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828287.220908:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828287/real 1787828287] req@ffff907d908a2580 x1874670011665280/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828287.220921:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011665280/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828287.220929:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828287.220932:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828287.220935:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011665280:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828287.220954:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011665408:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828287.220978:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828287.220992:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828287/real 1787828287] req@ffff907d908a21c0 x1874670011665408/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828287.221005:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011665408/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828287.221010:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828287.221013:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828287.221015:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011665408:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828287.221019:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011665536:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828287.221035:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828287.221042:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828287/real 1787828287] req@ffff907d908a0f00 x1874670011665536/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828287.221051:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011665536/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828287.221060:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828287.221063:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828287.221066:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011665536:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828287.221072:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011665664:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828287.221093:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828287.221105:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828287/real 1787828287] req@ffff907d908a12c0 x1874670011665664/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828287.221116:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011665664/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828287.221122:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828287.221124:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828287.221126:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011665664:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828287.348676:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828287.348731:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828288.372714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828288.372775:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828289.396673:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828289.396743:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828290.420705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828290.420755:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828291.444688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828291.444766:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828292.340674:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828292.340682:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828292.340686:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828292.340694:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3696 00000100:00080000:1.0:1787828292.340704:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828292.340709:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828292.340743:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011665792/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828292.340765:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828292.340768:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828292.340769:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828292.340772:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3696 00000100:00080000:1.0:1787828292.340774:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828292.340777:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828292.340784:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828292.340793:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011665920/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828292.340801:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828292.340803:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828292.340805:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828292.340808:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3696 00000100:00080000:1.0:1787828292.340810:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828292.340812:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828292.340818:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011665792:192.168.203.104@tcp:250:kworker.0 00000008:00100000:1.0:1787828292.340819:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828292.340831:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011666048/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828292.340838:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828292.340841:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828292.340842:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828292.340844:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3696 00000100:00080000:1.0:1787828292.340847:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828292.340849:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828292.340852:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828292.340860:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011666176/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828292.340890:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828292.340928:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828292/real 1787828292] req@ffff907d908a3840 x1874670011665792/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828292.340942:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011665792/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828292.340950:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828292.340953:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828292.340956:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011665792:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828292.340961:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011665920:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828292.340977:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828292.340985:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828292/real 1787828292] req@ffff907d908a1680 x1874670011665920/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828292.340993:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011665920/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828292.340999:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828292.341000:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828292.341002:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011665920:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828292.341025:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011666048:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828292.341045:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828292.341058:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828292/real 1787828292] req@ffff907d908a12c0 x1874670011666048/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828292.341071:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011666048/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828292.341078:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828292.341080:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828292.341082:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011666048:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828292.341087:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011666176:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828292.341111:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828292.341123:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828292/real 1787828292] req@ffff907d908a0f00 x1874670011666176/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828292.341130:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011666176/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828292.341140:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828292.341142:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828292.341144:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011666176:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828292.468711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828292.468770:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828293.492679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828293.492772:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828294.516703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828294.516812:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828295.540668:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828295.540812:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828296.564696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828296.564741:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828297.460774:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828297.460784:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828297.460789:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828297.460798:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3701 00000100:00080000:3.0:1787828297.460810:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828297.460817:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828297.460852:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011666304/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828297.460883:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828297.460889:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828297.460892:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828297.460896:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3701 00000100:00080000:3.0:1787828297.460899:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828297.460904:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828297.460914:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828297.460923:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011666304:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828297.460931:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011666432/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828297.460944:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828297.460948:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828297.460950:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828297.460954:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3701 00000100:00080000:3.0:1787828297.460957:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828297.460961:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828297.460967:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828297.460978:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011666560/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828297.460985:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787828297.460993:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828297.460998:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828297.461000:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828297.461005:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3701 00000100:00080000:3.0:1787828297.461009:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828297.461013:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828297.461019:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828297.461023:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828297/real 1787828297] req@ffff907da562d680 x1874670011666304/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828297.461042:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011666688/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828297.461044:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011666304/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828297.461073:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828297.461079:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828297.461083:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011666304:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828297.461106:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011666432:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828297.461126:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828297.461135:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828297/real 1787828297] req@ffff907da562cf00 x1874670011666432/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828297.461144:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011666432/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828297.461153:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828297.461157:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828297.461160:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011666432:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828297.461167:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011666560:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828297.461188:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828297.461201:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828297/real 1787828297] req@ffff907da562ed00 x1874670011666560/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828297.461210:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011666560/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828297.461216:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828297.461218:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828297.461220:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011666560:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828297.461224:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011666688:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828297.461238:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828297.461244:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828297/real 1787828297] req@ffff907da562c780 x1874670011666688/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828297.461252:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011666688/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828297.461256:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828297.461258:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828297.461261:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011666688:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828297.588724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828297.588794:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828298.612679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828298.612743:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828299.636707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828299.636824:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828300.660760:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828300.660779:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787828301.684668:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828301.684747:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828302.580712:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828302.580719:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828302.580723:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828302.580732:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3706 00000100:00080000:0.0:1787828302.580742:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828302.580748:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828302.580777:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011666816/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828302.580799:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828302.580802:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828302.580804:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828302.580807:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3706 00000100:00080000:0.0:1787828302.580810:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828302.580813:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828302.580823:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828302.580840:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011666944/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828302.580849:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011666816:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828302.580859:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828302.580865:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828302.580867:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828302.580872:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3706 00000100:00080000:0.0:1787828302.580875:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828302.580878:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828302.580883:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828302.580895:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011667072/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828302.580902:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828302.580905:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828302.580908:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787828302.580909:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787828302.580916:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3706 00000100:00080000:0.0:1787828302.580919:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828302.580922:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828302.580926:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828302.580934:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011667200/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828302.580948:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828302/real 1787828302] req@ffff907d908443c0 x1874670011666816/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828302.580963:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011666816/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828302.580974:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828302.580977:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828302.580981:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011666816:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828302.581001:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011666944:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787828302.581019:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828302.581028:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828302/real 1787828302] req@ffff907d90844b40 x1874670011666944/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828302.581036:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011666944/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828302.581042:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828302.581045:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828302.581048:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011666944:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828302.581053:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011667072:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828302.581065:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828302.581071:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828302/real 1787828302] req@ffff907d90845e00 x1874670011667072/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828302.581079:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011667072/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828302.581084:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828302.581086:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828302.581088:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011667072:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828302.581091:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011667200:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828302.581102:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828302.581107:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828302/real 1787828302] req@ffff907d908470c0 x1874670011667200/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828302.581115:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011667200/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828302.581120:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828302.581122:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828302.581124:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011667200:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828302.708703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828302.708760:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828303.732728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828303.732815:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828304.756718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828304.756751:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828305.780756:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828305.780806:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787828306.804690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828306.804766:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828307.700738:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828307.700751:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828307.700757:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828307.700771:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3711 00000100:00080000:3.0:1787828307.700793:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828307.700801:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828307.700842:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011667328/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828307.700867:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828307.700870:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828307.700872:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828307.700876:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3711 00000100:00080000:3.0:1787828307.700880:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828307.700885:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828307.700899:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828307.700917:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011667456/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828307.700922:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011667328:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828307.700952:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828307.700955:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828307.700957:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828307.700962:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3711 00000100:00080000:3.0:1787828307.700966:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828307.700970:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828307.700976:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828307.700990:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011667584/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828307.700997:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828307.701000:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828307.701002:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828307.701004:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3711 00000100:00080000:3.0:1787828307.701008:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787828307.701013:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787828307.701014:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828307.701027:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828307.701038:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011667712/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828307.701073:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828307/real 1787828307] req@ffff907da562f480 x1874670011667328/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828307.701094:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011667328/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828307.701120:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828307.701125:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828307.701130:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011667328:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828307.701154:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011667456:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787828307.701179:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828307.701192:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828307/real 1787828307] req@ffff907da562c000 x1874670011667456/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828307.701206:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011667456/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828307.701215:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828307.701219:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828307.701222:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011667456:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828307.701230:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011667584:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828307.701253:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828307.701265:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828307/real 1787828307] req@ffff907da562e580 x1874670011667584/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828307.701279:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011667584/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828307.701288:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828307.701292:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828307.701296:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011667584:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828307.701303:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011667712:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828307.701320:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828307.701330:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828307/real 1787828307] req@ffff907da562d2c0 x1874670011667712/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828307.701361:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011667712/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828307.701370:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828307.701373:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828307.701377:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011667712:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828307.828739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828307.828755:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828308.852713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828308.852798:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828309.876742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828309.876878:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828310.900688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828310.900721:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828311.924703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828311.924779:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828312.820676:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828312.820687:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828312.820692:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828312.820704:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3716 00000100:00080000:3.0:1787828312.820717:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828312.820726:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828312.820774:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011667840/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828312.820802:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828312.820805:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828312.820806:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828312.820810:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3716 00000100:00080000:3.0:1787828312.820812:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828312.820815:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828312.820826:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828312.820835:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011667968/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828312.820844:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828312.820847:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828312.820849:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828312.820852:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3716 00000100:00080000:3.0:1787828312.820855:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828312.820859:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828312.820864:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828312.820876:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011668096/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828312.820880:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011667840:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828312.820891:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828312.820894:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828312.820896:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828312.820899:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3716 00000100:00080000:3.0:1787828312.820901:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828312.820905:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828312.820909:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828312.820918:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011668224/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828312.820996:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011667968:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828312.821029:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011668096:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828312.821048:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011668224:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828312.948707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828312.948773:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828313.972698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828313.972753:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828314.996690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828314.996752:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787828315.892983:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787828315.893031:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787828315.893048:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828315.893087:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828315.893097:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828315.893107:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:2.0:1787828315.893136:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828312/real 1787828315] req@ffff907da562fc00 x1874670011668096/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828315.893165:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011668096/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828315.893177:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828315.893182:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828315.893187:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011668096:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828315.893195:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828312/real 1787828315] req@ffff907da562e940 x1874670011668224/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828315.893210:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011668224/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828315.893224:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828315.893228:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828315.893232:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011668224:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828315.893238:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828312/real 1787828315] req@ffff907da562c3c0 x1874670011667840/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828315.893252:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011667840/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828315.893260:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828315.893263:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828315.893267:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011667840:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828315.893294:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828312/real 1787828315] req@ffff907da562f0c0 x1874670011667968/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828315.893311:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011667968/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828315.893320:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828315.893324:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828315.893328:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011667968:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828316.020711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828316.020804:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828317.044725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828317.044791:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828317.940675:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828317.940687:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828317.940692:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828317.940713:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3721 00000100:00080000:3.0:1787828317.940722:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828317.940729:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828317.940757:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011668352/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828317.940793:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828317.940796:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828317.940797:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828317.940800:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3721 00000100:00080000:3.0:1787828317.940802:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828317.940805:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828317.940813:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828317.940822:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011668480/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828317.940829:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011668352:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828317.940842:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828317.940848:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828317.940850:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828317.940853:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3721 00000100:00080000:3.0:1787828317.940855:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828317.940857:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828317.940860:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828317.940867:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011668608/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828317.940874:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828317.940876:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828317.940877:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828317.940880:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3721 00000100:00080000:3.0:1787828317.940896:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828317.940900:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:2.0:1787828317.940905:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:3.0:1787828317.940908:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828317.940924:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011668736/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828317.940940:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828317/real 1787828317] req@ffff907da562da40 x1874670011668352/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828317.940956:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011668352/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828317.940967:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828317.940971:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828317.940975:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011668352:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828317.940999:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011668480:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828317.941023:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828317.941038:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828317/real 1787828317] req@ffff907da562f840 x1874670011668480/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828317.941050:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011668480/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828317.941058:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828317.941060:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828317.941064:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011668480:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828317.941070:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011668608:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828317.941097:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828317.941109:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828317/real 1787828317] req@ffff907da562e1c0 x1874670011668608/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828317.941121:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011668608/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828317.941137:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828317.941141:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828317.941144:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011668608:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828317.941150:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011668736:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828317.941170:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828317.941182:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828317/real 1787828317] req@ffff907da562cb40 x1874670011668736/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828317.941193:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011668736/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828317.941202:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828317.941205:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828317.941209:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011668736:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828318.068734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828318.068772:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828319.092741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828319.092966:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828320.116757:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828320.116846:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828321.140761:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828321.140763:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828322.164824:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828322.164907:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828323.060703:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828323.060711:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828323.060715:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828323.060723:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3726 00000100:00080000:1.0:1787828323.060732:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828323.060738:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828323.060770:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011668864/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828323.060791:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828323.060794:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828323.060795:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828323.060798:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3726 00000100:00080000:1.0:1787828323.060800:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828323.060802:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828323.060810:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828323.060819:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011668992/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828323.060827:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828323.060830:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828323.060831:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828323.060836:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3726 00000100:00080000:1.0:1787828323.060839:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828323.060841:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828323.060844:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828323.060854:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011669120/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828323.060863:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011668864:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828323.060866:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828323.060878:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828323.060884:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828323.060889:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3726 00000100:00080000:1.0:1787828323.060891:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828323.060892:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828323.060895:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828323.060907:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011669248/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828323.060971:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828323.061026:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828323/real 1787828323] req@ffff907d908a21c0 x1874670011668864/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828323.061046:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011668864/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828323.061059:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828323.061063:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828323.061068:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011668864:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828323.061076:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011668992:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828323.061097:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828323.061107:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828323/real 1787828323] req@ffff907d908a2580 x1874670011668992/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828323.061117:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011668992/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828323.061125:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828323.061127:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828323.061130:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011668992:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828323.061158:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011669120:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828323.061174:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828323.061183:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828323/real 1787828323] req@ffff907d908a2940 x1874670011669120/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828323.061192:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011669120/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828323.061200:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828323.061202:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828323.061205:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011669120:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828323.061211:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011669248:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828323.061243:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828323.061254:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828323/real 1787828323] req@ffff907d908a0000 x1874670011669248/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828323.061264:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011669248/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828323.061272:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828323.061275:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828323.061278:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011669248:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828323.188708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828323.188736:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828324.212673:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828324.212941:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828325.236726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828325.236808:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828326.260819:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828326.260918:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787828327.284702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828327.284745:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828328.180705:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828328.180718:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828328.180724:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828328.180737:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3731 00000100:00080000:1.0:1787828328.180753:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828328.180762:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828328.180808:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011669376/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828328.180846:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828328.180851:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828328.180853:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828328.180858:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3731 00000100:00080000:1.0:1787828328.180862:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828328.180866:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828328.180878:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828328.180894:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011669504/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828328.180910:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828328.180914:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828328.180917:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828328.180927:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011669376:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828328.180929:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3731 00000100:00080000:1.0:1787828328.180938:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828328.180946:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828328.180952:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828328.180970:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011669632/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828328.180984:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828328.180989:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828328.180992:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828328.180996:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3731 00000100:00080000:1.0:1787828328.181002:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828328.181007:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828328.181013:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787828328.181024:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828328.181032:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011669760/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828328.181102:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828328/real 1787828328] req@ffff907d908a1e00 x1874670011669376/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828328.181127:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011669376/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828328.181141:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828328.181145:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828328.181151:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011669376:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828328.181181:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011669504:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828328.181246:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828328.181269:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828328/real 1787828328] req@ffff907d908a2d00 x1874670011669504/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828328.181285:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011669504/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828328.181296:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828328.181300:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828328.181305:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011669504:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828328.181313:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011669632:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828328.181335:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828328.181346:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828328/real 1787828328] req@ffff907d908a0000 x1874670011669632/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828328.181360:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011669632/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828328.181370:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828328.181373:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828328.181377:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011669632:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828328.181384:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011669760:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828328.181402:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828328.181413:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828328/real 1787828328] req@ffff907d908a2940 x1874670011669760/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828328.181428:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011669760/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828328.181437:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828328.181441:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828328.181444:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011669760:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828328.308675:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828328.308735:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828329.332733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828329.332991:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828330.356737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828330.356930:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828331.380700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828331.380841:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828332.404772:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828332.404865:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828333.300694:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828333.300703:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828333.300706:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828333.300715:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3737 00000100:00080000:1.0:1787828333.300725:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828333.300731:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828333.300772:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011669888/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828333.300796:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828333.300799:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828333.300800:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828333.300803:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3737 00000100:00080000:1.0:1787828333.300805:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828333.300808:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828333.300816:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828333.300825:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011670016/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828333.300832:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011669888:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828333.300837:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828333.300841:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828333.300842:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828333.300845:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3737 00000100:00080000:1.0:1787828333.300848:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828333.300850:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828333.300853:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828333.300861:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011670144/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828333.300867:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828333.300870:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828333.300872:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828333.300874:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3737 00000100:00080000:1.0:1787828333.300876:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828333.300889:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828333.300892:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828333.300899:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011670272/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828333.300912:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828333.300953:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828333/real 1787828333] req@ffff907d908a2580 x1874670011669888/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828333.300967:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011669888/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828333.300975:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828333.300978:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828333.300981:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011669888:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828333.301001:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011670016:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828333.301025:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828333.301051:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828333/real 1787828333] req@ffff907d908a21c0 x1874670011670016/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828333.301064:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011670016/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828333.301070:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828333.301072:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828333.301075:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011670016:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828333.301079:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011670144:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828333.301095:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828333.301103:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828333/real 1787828333] req@ffff907d908a0f00 x1874670011670144/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828333.301118:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011670144/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828333.301124:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828333.301126:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828333.301128:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011670144:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828333.301132:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011670272:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828333.301142:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828333.301148:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828333/real 1787828333] req@ffff907d908a12c0 x1874670011670272/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828333.301170:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011670272/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828333.301177:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828333.301179:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828333.301181:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011670272:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828333.428705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828333.428807:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828334.452687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828334.452773:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828335.476758:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828335.476843:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828336.500742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828336.500991:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828337.524703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828337.524807:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828338.420725:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828338.420733:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828338.420737:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828338.420745:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3742 00000100:00080000:0.0:1787828338.420754:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828338.420759:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828338.420799:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011670400/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828338.420829:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828338.420834:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828338.420836:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828338.420846:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3742 00000100:00080000:0.0:1787828338.420852:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828338.420857:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828338.420868:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828338.420874:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011670400:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828338.420889:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011670528/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828338.420903:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828338.420906:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828338.420909:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828338.420913:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3742 00000100:00080000:0.0:1787828338.420920:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828338.420924:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828338.420930:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787828338.420931:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828338.420949:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011670656/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828338.420961:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828338.420965:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828338.420968:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828338.420972:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3742 00000100:00080000:0.0:1787828338.420976:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787828338.420980:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828338/real 1787828338] req@ffff907d90847c00 x1874670011670400/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828338.420983:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828338.420991:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828338.421005:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011670784/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828338.421010:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011670400/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828338.421039:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828338.421043:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828338.421047:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011670400:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828338.421068:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011670528:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787828338.421085:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828338.421094:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828338/real 1787828338] req@ffff907d90846580 x1874670011670528/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828338.421102:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011670528/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828338.421107:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828338.421109:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828338.421112:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011670528:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828338.421115:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011670656:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828338.421146:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828338.421175:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828338/real 1787828338] req@ffff907d908470c0 x1874670011670656/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828338.421187:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011670656/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828338.421203:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828338.421206:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828338.421209:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011670656:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828338.421215:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011670784:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828338.421234:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828338.421243:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828338/real 1787828338] req@ffff907d90845e00 x1874670011670784/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828338.421254:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011670784/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828338.421262:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828338.421265:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828338.421268:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011670784:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828338.548686:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828338.548752:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828339.572698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828339.572859:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828340.596705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828340.596863:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828341.620824:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828341.620918:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828342.644727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828342.644846:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828343.540691:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828343.540701:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828343.540705:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828343.540713:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3747 00000100:00080000:2.0:1787828343.540722:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828343.540728:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828343.540759:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011670912/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828343.540786:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828343.540789:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828343.540790:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828343.540817:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3747 00000100:00080000:2.0:1787828343.540821:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828343.540824:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828343.540834:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828343.540846:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011671040/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828343.540855:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828343.540859:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828343.540860:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828343.540864:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3747 00000100:00080000:2.0:1787828343.540866:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828343.540869:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828343.540874:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828343.540880:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011670912:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828343.540886:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011671168/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828343.540898:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828343.540901:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828343.540903:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828343.540907:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3747 00000100:00080000:2.0:1787828343.540910:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828343.540913:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828343.540917:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828343.540925:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011671296/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828343.540963:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828343.541022:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828343/real 1787828343] req@ffff907d911492c0 x1874670011670912/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828343.541042:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011670912/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828343.541054:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828343.541059:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828343.541062:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011670912:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828343.541068:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011671040:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787828343.541101:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828343.541111:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828343/real 1787828343] req@ffff907d91148f00 x1874670011671040/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828343.541139:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011671040/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828343.541147:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828343.541150:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828343.541152:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011671040:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828343.541174:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011671168:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828343.541191:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828343.541200:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828343/real 1787828343] req@ffff907d9114a940 x1874670011671168/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828343.541210:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011671168/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828343.541217:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828343.541219:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828343.541222:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011671168:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828343.541226:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011671296:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828343.541238:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828343.541246:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828343/real 1787828343] req@ffff907d91148b40 x1874670011671296/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828343.541256:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011671296/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828343.541262:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828343.541264:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828343.541266:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011671296:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828343.668719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828343.668762:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828344.692740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828344.692829:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828345.716729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828345.716933:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828346.740717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828346.740929:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828347.764796:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828347.764923:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828348.660895:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828348.660909:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828348.660916:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828348.660933:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3752 00000100:00080000:1.0:1787828348.660952:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828348.660964:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828348.661027:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011671424/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828348.661112:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828348.661116:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828348.661118:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828348.661139:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3752 00000100:00080000:1.0:1787828348.661145:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828348.661151:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828348.661168:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828348.661193:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011671552/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828348.661220:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828348.661231:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828348.661235:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787828348.661235:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011671424:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828348.661262:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3752 00000100:00080000:1.0:1787828348.661268:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828348.661273:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828348.661281:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828348.661303:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011671680/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828348.661315:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828348.661320:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828348.661323:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828348.661327:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3752 00000100:00080000:1.0:1787828348.661332:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828348.661338:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828348.661344:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787828348.661354:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828348.661367:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011671808/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828348.661434:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828348/real 1787828348] req@ffff907d908a1680 x1874670011671424/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828348.661471:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011671424/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828348.661493:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828348.661502:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828348.661510:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011671424:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828348.661554:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011671552:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828348.661590:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828348.661666:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828348/real 1787828348] req@ffff907d908a3840 x1874670011671552/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828348.661706:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011671552/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828348.661725:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828348.661732:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828348.661740:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011671552:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828348.661751:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011671680:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828348.661785:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828348.661808:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828348/real 1787828348] req@ffff907d908a12c0 x1874670011671680/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828348.661832:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011671680/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828348.661872:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828348.661880:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828348.661887:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011671680:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828348.661900:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011671808:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828348.661941:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828348.661957:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828348/real 1787828348] req@ffff907d908a0f00 x1874670011671808/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828348.661971:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011671808/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828348.661982:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828348.661986:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828348.661990:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011671808:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828348.788720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828348.788838:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828349.812713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828349.812808:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828350.836774:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828350.836866:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828351.860672:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828351.860908:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828352.884709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828352.884922:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828353.780802:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828353.780816:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828353.780833:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828353.780846:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3757 00000100:00080000:1.0:1787828353.780861:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828353.780870:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828353.780929:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011671936/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828353.780968:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828353.780972:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828353.780975:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828353.780980:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3757 00000100:00080000:1.0:1787828353.780984:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828353.780988:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828353.781001:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828353.781059:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011672064/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828353.781079:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828353.781085:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828353.781090:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828353.781099:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3757 00000100:00100000:3.0:1787828353.781099:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011671936:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828353.781120:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828353.781130:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828353.781138:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828353.781166:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011672192/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828353.781178:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828353.781183:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828353.781186:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828353.781191:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3757 00000100:00080000:1.0:1787828353.781195:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828353.781202:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828353.781211:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828353.781236:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011672320/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828353.781245:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828353.781314:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828353/real 1787828353] req@ffff907d908a21c0 x1874670011671936/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828353.781333:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011671936/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828353.781345:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828353.781348:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828353.781352:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011671936:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828353.781381:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011672064:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828353.781399:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828353.781409:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828353/real 1787828353] req@ffff907d908a2580 x1874670011672064/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828353.781417:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011672064/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828353.781423:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828353.781426:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828353.781428:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011672064:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828353.781432:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011672192:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828353.781445:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828353.781450:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828353/real 1787828353] req@ffff907d908a2940 x1874670011672192/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828353.781458:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011672192/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828353.781464:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828353.781466:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828353.781468:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011672192:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828353.781473:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011672320:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828353.781484:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828353.781489:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828353/real 1787828353] req@ffff907d908a0000 x1874670011672320/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828353.781497:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011672320/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828353.781503:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828353.781504:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828353.781506:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011672320:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828353.908823:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828353.908877:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828354.932778:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828354.932926:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828355.956737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828355.956829:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828356.980785:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828356.980922:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828358.004733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828358.004840:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828358.900842:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828358.900862:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828358.900871:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828358.900890:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3762 00000100:00080000:2.0:1787828358.900909:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828358.900919:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828358.901008:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011672448/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828358.901053:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828358.901058:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828358.901062:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828358.901074:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3762 00000100:00080000:2.0:1787828358.901080:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828358.901085:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828358.901101:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828358.901122:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011672576/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828358.901138:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828358.901143:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828358.901145:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828358.901154:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3762 00000100:00100000:3.0:1787828358.901162:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011672448:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828358.901170:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828358.901182:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828358.901189:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828358.901205:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011672704/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828358.901218:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828358.901222:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828358.901225:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828358.901230:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3762 00000100:00080000:2.0:1787828358.901234:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828358.901238:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828358.901244:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828358.901259:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011672832/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828358.901297:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828358.901381:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828358/real 1787828358] req@ffff907d91148b40 x1874670011672448/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828358.901447:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011672448/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828358.901470:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828358.901481:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828358.901489:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011672448:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828358.901533:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011672576:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828358.901583:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828358.901656:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828358/real 1787828358] req@ffff907d9114a940 x1874670011672576/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828358.901687:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011672576/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828358.901705:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828358.901713:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828358.901721:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011672576:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828358.901735:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011672704:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828358.901778:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828358.901802:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828358/real 1787828358] req@ffff907d91148f00 x1874670011672704/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828358.901827:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011672704/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828358.901855:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828358.901862:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828358.901869:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011672704:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828358.901882:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011672832:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828358.901917:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828358.901937:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828358/real 1787828358] req@ffff907d911492c0 x1874670011672832/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828358.901962:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011672832/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828358.902007:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828358.902015:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828358.902023:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011672832:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828359.028772:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828359.028837:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828360.052713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828360.052773:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828361.076804:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828361.076942:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828362.100702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828362.100864:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828363.124666:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828363.124713:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828364.020719:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828364.020728:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828364.020732:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828364.020742:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3767 00000100:00080000:2.0:1787828364.020752:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828364.020759:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828364.020808:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011672960/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828364.020827:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828364.020830:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828364.020832:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828364.020835:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3767 00000100:00080000:2.0:1787828364.020838:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828364.020840:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828364.020865:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828364.020872:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011672960:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828364.020880:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011673088/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828364.020891:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828364.020894:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828364.020896:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828364.020898:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3767 00000100:00080000:2.0:1787828364.020902:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828364.020906:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828364.020911:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828364.020921:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011673216/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828364.020960:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828364.020963:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828364.020965:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1787828364.020969:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787828364.020970:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3767 00000100:00080000:2.0:1787828364.020976:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828364.020980:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828364.020985:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828364.021012:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011673344/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828364.021043:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828364/real 1787828364] req@ffff907d9114a580 x1874670011672960/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828364.021064:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011672960/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828364.021074:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828364.021078:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828364.021081:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011672960:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828364.021102:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011673088:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828364.021124:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828364.021135:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828364/real 1787828364] req@ffff907d9114ad00 x1874670011673088/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828364.021144:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011673088/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828364.021151:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828364.021154:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828364.021156:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011673088:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828364.021162:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011673216:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828364.021176:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828364.021183:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828364/real 1787828364] req@ffff907d91149a40 x1874670011673216/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828364.021191:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011673216/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828364.021198:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828364.021200:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828364.021203:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011673216:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828364.021207:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011673344:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828364.021218:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828364.021225:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828364/real 1787828364] req@ffff907d9114b840 x1874670011673344/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828364.021239:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011673344/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828364.021245:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828364.021247:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828364.021249:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011673344:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828364.148685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828364.148742:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828365.172745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828365.172823:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828366.196703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828366.196756:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828367.220699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828367.220826:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828368.244747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828368.244902:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828369.140693:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828369.140702:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828369.140705:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828369.140714:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3772 00000100:00080000:3.0:1787828369.140723:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828369.140729:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828369.140770:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011673472/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828369.140791:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828369.140794:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828369.140795:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828369.140798:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3772 00000100:00080000:3.0:1787828369.140801:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828369.140804:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828369.140812:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828369.140823:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011673600/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828369.140831:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828369.140834:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828369.140837:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828369.140839:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011673472:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828369.140842:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3772 00000100:00080000:3.0:1787828369.140849:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828369.140852:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828369.140855:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828369.140864:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011673728/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828369.140871:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828369.140874:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828369.140875:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828369.140877:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3772 00000100:00080000:3.0:1787828369.140880:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828369.140883:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828369.140885:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828369.140893:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011673856/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828369.140903:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828369.140954:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828369/real 1787828369] req@ffff907da562de00 x1874670011673472/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828369.140969:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011673472/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828369.140977:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828369.140980:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828369.140984:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011673472:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828369.141002:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011673600:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787828369.141019:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828369.141027:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828369/real 1787828369] req@ffff907da562cb40 x1874670011673600/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828369.141035:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011673600/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828369.141041:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828369.141043:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828369.141045:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011673600:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828369.141049:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011673728:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828369.141060:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828369.141066:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828369/real 1787828369] req@ffff907da562e1c0 x1874670011673728/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828369.141073:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011673728/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828369.141078:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828369.141080:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828369.141082:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011673728:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828369.141086:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011673856:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828369.141095:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828369.141102:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828369/real 1787828369] req@ffff907da562f840 x1874670011673856/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828369.141109:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011673856/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828369.141114:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828369.141116:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828369.141118:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011673856:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828369.268700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828369.268786:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828370.292716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828370.292814:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828371.316695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828371.316766:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828372.340695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828372.340780:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828373.364762:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828373.364868:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828374.260673:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828374.260682:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828374.260686:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828374.260695:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3778 00000100:00080000:3.0:1787828374.260704:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828374.260711:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828374.260764:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011673984/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828374.260800:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828374.260804:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828374.260806:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828374.260811:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3778 00000100:00080000:3.0:1787828374.260815:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828374.260819:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828374.260829:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828374.260843:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011674112/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828374.260855:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828374.260859:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828374.260861:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828374.260866:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3778 00000100:00080000:3.0:1787828374.260911:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828374.260917:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828374.260923:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828374.260937:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011674240/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828374.260945:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011673984:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828374.260953:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828374.260958:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828374.260961:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828374.260965:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3778 00000100:00080000:3.0:1787828374.260969:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828374.260972:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828374.260977:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828374.260992:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011674368/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828374.261022:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828374.261068:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828374/real 1787828374] req@ffff907da562da40 x1874670011673984/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828374.261098:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011673984/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828374.261114:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828374.261121:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828374.261129:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011673984:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828374.261145:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011674112:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828374.261178:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828374.261193:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828374/real 1787828374] req@ffff907da562f0c0 x1874670011674112/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828374.261205:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011674112/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828374.261213:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828374.261217:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828374.261220:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011674112:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828374.261245:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011674240:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828374.261266:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828374.261279:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828374/real 1787828374] req@ffff907da562c3c0 x1874670011674240/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828374.261290:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011674240/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828374.261297:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828374.261302:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828374.261306:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011674240:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828374.261313:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011674368:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828374.261332:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828374.261343:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828374/real 1787828374] req@ffff907da562e940 x1874670011674368/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828374.261356:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011674368/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828374.261364:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828374.261367:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828374.261370:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011674368:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828374.388736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828374.388826:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828375.412714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828375.412842:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828376.436746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828376.436921:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828377.460669:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828377.460747:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828378.484685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828378.484736:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828379.380767:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828379.380778:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828379.380783:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828379.380810:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3783 00000100:00080000:2.0:1787828379.380824:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828379.380856:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828379.380922:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011674496/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828379.380966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828379.380971:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828379.380973:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828379.380977:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3783 00000100:00080000:2.0:1787828379.380981:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828379.380986:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828379.381014:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828379.381036:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011674624/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828379.381048:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828379.381053:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011674496:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828379.381054:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828379.381057:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828379.381068:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3783 00000100:00080000:2.0:1787828379.381073:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828379.381076:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828379.381081:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828379.381095:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011674752/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828379.381105:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828379.381109:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828379.381111:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828379.381114:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3783 00000100:00080000:2.0:1787828379.381120:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828379.381124:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828379.381129:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828379.381140:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011674880/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828379.381193:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011674624:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828379.381229:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011674752:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828379.381253:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011674880:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828379.508742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828379.508906:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828380.532777:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828380.532822:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828381.556715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828381.556729:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787828382.452924:0:2337:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787828382.452987:0:2337:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787828382.453010:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828382.453062:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828382.453087:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828382.453099:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:0.0:1787828382.453161:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828379/real 1787828382] req@ffff907d911483c0 x1874670011674624/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828382.453204:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011674624/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828382.453222:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828382.453229:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828382.453236:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011674624:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828382.453248:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828379/real 1787828382] req@ffff907d9114b480 x1874670011674752/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828382.453266:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011674752/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828382.453288:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828382.453293:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828382.453297:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011674752:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828382.453307:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828379/real 1787828382] req@ffff907d91149680 x1874670011674880/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828382.453323:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011674880/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828382.453344:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828382.453350:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828382.453355:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011674880:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828382.453364:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828379/real 1787828382] req@ffff907d9114bc00 x1874670011674496/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828382.453382:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011674496/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828382.453395:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828382.453400:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828382.453405:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011674496:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828382.580686:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828382.580747:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828383.604735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828383.604863:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828384.500757:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828384.500769:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828384.500773:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828384.500785:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3788 00000100:00080000:0.0:1787828384.500796:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828384.500822:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828384.500863:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011675008/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828384.500899:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828384.500906:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828384.500910:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828384.500917:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3788 00000100:00080000:0.0:1787828384.500924:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828384.500947:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828384.500958:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828384.500960:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011675008:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828384.500978:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011675136/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828384.500992:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828384.500996:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828384.500998:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828384.501003:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3788 00000100:00080000:0.0:1787828384.501007:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828384.501012:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828384.501018:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828384.501032:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011675264/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828384.501040:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828384.501043:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828384.501045:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787828384.501051:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787828384.501053:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3788 00000100:00080000:0.0:1787828384.501059:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828384.501063:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828384.501067:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828384.501078:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011675392/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828384.501095:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828384/real 1787828384] req@ffff907d90844b40 x1874670011675008/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828384.501115:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011675008/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828384.501128:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828384.501134:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828384.501139:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011675008:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828384.501171:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011675136:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787828384.501199:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828384.501214:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828384/real 1787828384] req@ffff907d908443c0 x1874670011675136/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828384.501228:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011675136/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828384.501237:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828384.501241:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828384.501245:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011675136:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828384.501252:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011675264:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828384.501276:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828384.501290:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828384/real 1787828384] req@ffff907d90844000 x1874670011675264/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828384.501304:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011675264/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828384.501322:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828384.501325:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828384.501329:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011675264:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828384.501336:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011675392:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828384.501360:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828384.501374:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828384/real 1787828384] req@ffff907d908452c0 x1874670011675392/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828384.501390:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011675392/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828384.501400:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828384.501404:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828384.501408:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011675392:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828384.628734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828384.628850:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828385.652692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828385.652740:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828386.676714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828386.676854:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828387.700756:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828387.700805:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828388.724763:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828388.724849:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828389.620709:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828389.620721:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828389.620725:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828389.620736:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3793 00000100:00080000:3.0:1787828389.620747:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828389.620756:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828389.620815:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011675520/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828389.620842:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828389.620847:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828389.620849:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828389.620854:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3793 00000100:00080000:3.0:1787828389.620859:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828389.620864:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828389.620875:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828389.620880:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011675520:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828389.620904:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011675648/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828389.620919:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828389.620923:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828389.620925:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828389.620930:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3793 00000100:00080000:3.0:1787828389.620935:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787828389.620936:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787828389.620945:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828389.620951:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828389.620972:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828389/real 1787828389] req@ffff907da562fc00 x1874670011675520/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828389.620979:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011675776/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828389.620995:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828389.620997:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011675520/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828389.621002:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828389.621006:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828389.621018:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3793 00000100:00080000:1.0:1787828389.621021:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828389.621023:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828389.621030:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00080000:3.0:1787828389.621032:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828389.621039:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011675520:192.168.203.104@tcp:250:kworker.0 00000008:00100000:3.0:1787828389.621040:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828389.621060:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011675904/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828389.621068:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011675648:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787828389.621094:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828389.621105:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828389/real 1787828389] req@ffff907da562d2c0 x1874670011675648/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828389.621126:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011675648/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828389.621135:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828389.621138:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828389.621142:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011675648:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828389.621149:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011675776:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828389.621171:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828389.621180:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828389/real 1787828389] req@ffff907da562e580 x1874670011675776/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828389.621188:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011675776/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828389.621194:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828389.621196:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828389.621199:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011675776:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828389.621208:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011675904:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828389.621220:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828389.621227:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828389/real 1787828389] req@ffff907da562c000 x1874670011675904/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828389.621234:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011675904/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828389.621239:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828389.621241:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828389.621243:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011675904:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828389.748735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828389.748839:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828390.772715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828390.772824:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828391.796827:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828391.796882:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828392.820816:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828392.820905:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828393.844828:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828393.844916:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828394.740820:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828394.740853:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828394.740859:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828394.740875:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3798 00000100:00080000:0.0:1787828394.740893:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828394.740922:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828394.740992:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011676032/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828394.741027:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828394.741032:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828394.741034:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828394.741039:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3798 00000100:00080000:0.0:1787828394.741045:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828394.741050:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828394.741062:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828394.741079:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011676160/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828394.741094:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828394.741099:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828394.741106:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828394.741107:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011676032:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828394.741116:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3798 00000100:00080000:0.0:1787828394.741123:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828394.741127:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828394.741144:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828394.741164:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011676288/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828394.741178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828394.741184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828394.741186:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828394.741191:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3798 00000100:00080000:0.0:1787828394.741196:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787828394.741201:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787828394.741209:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828394.741218:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828394.741254:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011676416/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828394.741287:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828394/real 1787828394] req@ffff907d90844780 x1874670011676032/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828394.741317:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011676032/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828394.741332:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828394.741337:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828394.741343:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011676032:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828394.741388:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011676160:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787828394.741421:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828394.741436:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828394/real 1787828394] req@ffff907d90845680 x1874670011676160/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828394.741450:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011676160/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828394.741460:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828394.741464:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828394.741468:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011676160:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828394.741474:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011676288:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828394.741493:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828394.741503:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828394/real 1787828394] req@ffff907d90846d00 x1874670011676288/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828394.741516:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011676288/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828394.741526:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828394.741529:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828394.741532:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011676288:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828394.741538:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011676416:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828394.741554:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828394.741565:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828394/real 1787828394] req@ffff907d90846940 x1874670011676416/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828394.741586:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011676416/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828394.741595:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828394.741598:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828394.741601:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011676416:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787828394.868817:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828394.868841:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:0.0:1787828395.892696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828395.892808:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828396.916689:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828396.916724:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828397.940826:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828397.940912:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828398.964744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828398.964772:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828399.860779:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828399.860788:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828399.860792:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828399.860801:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3803 00000100:00080000:1.0:1787828399.860811:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828399.860817:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828399.860851:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011676544/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828399.860877:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828399.860880:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828399.860881:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828399.860884:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3803 00000100:00080000:1.0:1787828399.860887:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828399.860891:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828399.860899:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828399.860910:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011676672/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828399.860919:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011676544:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828399.860921:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828399.860939:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828399.860942:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828399.860946:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3803 00000100:00080000:1.0:1787828399.860948:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828399.860950:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828399.860953:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828399.860963:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011676800/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828399.860970:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828399.860972:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828399.860974:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828399.860977:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3803 00000100:00080000:1.0:1787828399.860990:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828399.860993:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828399.860996:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787828399.861007:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828399.861008:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011676928/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828399.861065:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828399/real 1787828399] req@ffff907d908a2d00 x1874670011676544/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828399.861081:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011676544/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828399.861089:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828399.861092:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828399.861095:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011676544:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828399.861132:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011676672:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828399.861160:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828399.861170:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828399/real 1787828399] req@ffff907d908a1e00 x1874670011676672/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828399.861179:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011676672/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828399.861185:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828399.861187:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828399.861189:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011676672:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828399.861194:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011676800:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828399.861216:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828399.861222:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828399/real 1787828399] req@ffff907d908a0000 x1874670011676800/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828399.861230:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011676800/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828399.861235:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828399.861237:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828399.861239:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011676800:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828399.861243:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011676928:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828399.861254:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828399.861260:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828399/real 1787828399] req@ffff907d908a2940 x1874670011676928/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828399.861273:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011676928/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828399.861278:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828399.861280:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828399.861282:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011676928:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828399.988709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828399.988787:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828401.012776:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828401.012788:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828402.036744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828402.036832:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828403.060757:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828403.060781:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828404.084830:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828404.084866:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828404.980788:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828404.980796:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828404.980800:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828404.980808:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3808 00000100:00080000:3.0:1787828404.980817:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828404.980823:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828404.980855:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011677056/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828404.980879:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828404.980882:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828404.980883:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828404.980886:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3808 00000100:00080000:3.0:1787828404.980888:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828404.980890:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828404.980898:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828404.980907:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011677184/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828404.980926:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828404.980929:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828404.980930:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828404.980933:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3808 00000100:00080000:3.0:1787828404.980935:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828404.980938:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828404.980940:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828404.980949:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011677312/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828404.980958:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828404.980960:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828404.980962:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828404.980968:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3808 00000100:00080000:3.0:1787828404.980976:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828404.980997:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828404.980998:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011677056:192.168.203.104@tcp:250:kworker.0 00000008:00100000:3.0:1787828404.981004:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828404.981022:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011677440/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828404.981133:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828404.981193:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828404/real 1787828404] req@ffff907da562f480 x1874670011677056/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828404.981216:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011677056/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828404.981230:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828404.981235:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828404.981240:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011677056:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828404.981250:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011677184:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828404.981275:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828404.981288:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828404/real 1787828404] req@ffff907da562c780 x1874670011677184/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828404.981301:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011677184/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828404.981341:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828404.981344:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828404.981347:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011677184:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828404.981376:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011677312:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828404.981394:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828404.981402:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828404/real 1787828404] req@ffff907da562ed00 x1874670011677312/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828404.981433:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011677312/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828404.981442:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828404.981445:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828404.981449:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011677312:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828404.981455:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011677440:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828404.981473:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828404.981482:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828404/real 1787828404] req@ffff907da562cf00 x1874670011677440/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828404.981494:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011677440/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828404.981502:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828404.981505:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828404.981510:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011677440:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828405.108785:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828405.108890:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828406.132729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828406.132797:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828407.156730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828407.156799:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828408.180742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828408.180782:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828409.204751:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828409.204857:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828410.100747:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828410.100759:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828410.100765:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828410.100785:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3813 00000100:00080000:3.0:1787828410.100799:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828410.100821:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828410.100883:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011677568/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828410.100914:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828410.100919:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828410.100921:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828410.100925:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3813 00000100:00080000:3.0:1787828410.100930:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828410.100936:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828410.100948:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828410.100972:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011677568:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828410.100977:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011677696/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828410.101003:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828410.101010:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828410.101013:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828410.101021:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3813 00000100:00080000:3.0:1787828410.101028:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828410.101035:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828410.101044:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828410.101065:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011677824/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828410.101069:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787828410.101087:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828410.101094:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828410.101097:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828410.101104:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3813 00000100:00080000:3.0:1787828410.101109:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828410.101114:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828410.101138:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828410.101151:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828410/real 1787828410] req@ffff907da562d680 x1874670011677568/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828410.101162:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011677952/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828410.101211:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011677568/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828410.101233:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828410.101243:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828410.101252:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011677568:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828410.101292:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011677696:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828410.101347:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828410.101369:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828410/real 1787828410] req@ffff907da562cf00 x1874670011677696/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828410.101391:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011677696/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828410.101404:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828410.101410:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828410.101415:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011677696:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828410.101426:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011677824:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828410.101484:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828410.101513:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828410/real 1787828410] req@ffff907da562ed00 x1874670011677824/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828410.101538:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011677824/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828410.101556:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828410.101565:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828410.101574:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011677824:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828410.101590:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011677952:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828410.101738:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828410.101776:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828410/real 1787828410] req@ffff907da562c780 x1874670011677952/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828410.101803:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011677952/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828410.101820:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828410.101826:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828410.101832:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011677952:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828410.228695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828410.228739:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828411.252703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828411.252746:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828412.276712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828412.276812:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828413.300715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828413.300834:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828414.324720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828414.324809:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828415.220704:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828415.220718:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828415.220724:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828415.220741:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3818 00000100:00080000:3.0:1787828415.220759:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828415.220772:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828415.220846:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011678080/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828415.220883:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828415.220888:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828415.220891:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828415.220897:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3818 00000100:00080000:3.0:1787828415.220902:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828415.220908:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828415.220921:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828415.220940:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011678208/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828415.220954:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828415.220959:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828415.220962:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828415.220966:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3818 00000100:00080000:3.0:1787828415.220970:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828415.220980:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828415.220982:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011678080:192.168.203.104@tcp:250:kworker.0 00000008:00100000:3.0:1787828415.221004:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828415.221025:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011678336/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828415.221040:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828415.221045:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828415.221048:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828415.221052:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3818 00000100:00080000:3.0:1787828415.221068:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828415.221073:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828415.221079:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787828415.221089:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828415.221107:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011678464/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828415.221181:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828415/real 1787828415] req@ffff907da562f480 x1874670011678080/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828415.221208:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011678080/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828415.221223:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828415.221228:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828415.221234:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011678080:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828415.221245:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011678208:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828415.221279:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828415.221296:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828415/real 1787828415] req@ffff907da562c000 x1874670011678208/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828415.221315:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011678208/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828415.221332:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828415.221339:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828415.221345:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011678208:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828415.221400:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011678336:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828415.221446:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828415.221471:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828415/real 1787828415] req@ffff907da562e580 x1874670011678336/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828415.221496:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011678336/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828415.221515:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828415.221522:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828415.221528:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011678336:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828415.221539:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011678464:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828415.221569:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828415.221653:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828415/real 1787828415] req@ffff907da562d2c0 x1874670011678464/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828415.221686:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011678464/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828415.221698:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828415.221702:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828415.221706:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011678464:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828415.348712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828415.348785:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828416.372699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828416.372821:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828417.396786:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828417.396823:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828418.420697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828418.420752:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828419.444725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828419.444817:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828420.340779:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828420.340792:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828420.340815:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828420.340832:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3824 00000100:00080000:3.0:1787828420.340849:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828420.340859:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828420.340925:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011678592/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828420.340959:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828420.340963:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828420.340964:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828420.340968:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3824 00000100:00080000:3.0:1787828420.340971:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828420.340973:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828420.340984:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828420.340995:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011678720/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828420.341004:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828420.341032:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828420.341034:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828420.341039:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3824 00000100:00080000:3.0:1787828420.341042:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828420.341048:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828420.341056:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828420.341089:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011678592:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828420.341090:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011678848/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828420.341121:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828420.341124:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828420.341125:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828420.341130:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3824 00000100:00080000:3.0:1787828420.341132:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828420.341135:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828420.341141:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828420.341155:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011678976/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828420.341209:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828420.341274:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828420/real 1787828420] req@ffff907da562fc00 x1874670011678592/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828420.341307:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011678592/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828420.341321:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828420.341326:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828420.341342:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011678592:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828420.341352:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011678720:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828420.341377:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828420.341391:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828420/real 1787828420] req@ffff907da562e940 x1874670011678720/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828420.341416:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011678720/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828420.341426:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828420.341429:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828420.341433:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011678720:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828420.341474:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011678848:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828420.341493:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828420.341503:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828420/real 1787828420] req@ffff907da562c3c0 x1874670011678848/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828420.341516:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011678848/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828420.341525:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828420.341528:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828420.341531:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011678848:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828420.341537:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011678976:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828420.341577:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828420.341595:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828420/real 1787828420] req@ffff907da562f0c0 x1874670011678976/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828420.341687:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011678976/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828420.341710:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828420.341714:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828420.341718:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011678976:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828420.468738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828420.468887:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828421.492709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828421.492772:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828422.516707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828422.516760:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828423.540739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828423.540818:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828424.564742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828424.564778:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828425.460806:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828425.460814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828425.460819:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828425.460828:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3829 00000100:00080000:0.0:1787828425.460838:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828425.460846:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828425.460881:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011679104/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828425.460902:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828425.460905:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828425.460907:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828425.460910:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3829 00000100:00080000:0.0:1787828425.460913:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828425.460915:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828425.460923:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828425.460932:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011679232/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828425.460941:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828425.460944:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828425.460945:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828425.460950:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3829 00000100:00080000:0.0:1787828425.460959:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828425.460964:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011679104:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828425.460965:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828425.460974:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828425.460982:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011679360/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828425.460991:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828425.460994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828425.460995:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828425.460999:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3829 00000100:00080000:0.0:1787828425.461001:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828425.461004:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828425.461007:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828425.461016:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011679488/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828425.461029:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828425.461068:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828425/real 1787828425] req@ffff907d90844f00 x1874670011679104/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828425.461084:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011679104/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828425.461093:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828425.461096:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828425.461099:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011679104:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828425.461105:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011679232:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828425.461122:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828425.461131:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828425/real 1787828425] req@ffff907d90845a40 x1874670011679232/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828425.461146:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011679232/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828425.461152:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828425.461154:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828425.461157:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011679232:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828425.461175:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011679360:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828425.461189:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828425.461197:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828425/real 1787828425] req@ffff907d90847480 x1874670011679360/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828425.461206:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011679360/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828425.461212:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828425.461214:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828425.461216:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011679360:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828425.461220:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011679488:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828425.461241:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828425.461250:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828425/real 1787828425] req@ffff907d90846940 x1874670011679488/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828425.461258:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011679488/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828425.461269:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828425.461272:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828425.461274:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011679488:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828425.588717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828425.588731:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828426.612772:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828426.612881:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828427.636696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828427.636738:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828428.660755:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828428.660894:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828429.684664:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828429.684735:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828430.580672:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828430.580681:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828430.580684:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828430.580693:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3834 00000100:00080000:0.0:1787828430.580702:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828430.580710:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828430.580747:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011679616/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828430.580773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828430.580777:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828430.580779:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828430.580783:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3834 00000100:00080000:0.0:1787828430.580788:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828430.580792:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828430.580802:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828430.580809:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011679616:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828430.580819:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011679744/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828430.580832:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828430.580835:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828430.580852:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828430.580856:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3834 00000100:00080000:0.0:1787828430.580861:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828430.580864:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828430.580870:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787828430.580871:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828430.580888:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011679872/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828430.580900:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828430.580906:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1787828430.580907:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828430/real 1787828430] req@ffff907d90846d00 x1874670011679616/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828430.580910:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828430.580918:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3834 00000100:00080000:0.0:1787828430.580924:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828430.580929:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828430.580934:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011679616/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787828430.580935:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1787828430.580947:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787828430.580951:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011680000/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828430.580954:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828430.580962:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011679616:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828430.580995:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011679744:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828430.581013:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828430.581022:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828430/real 1787828430] req@ffff907d90845680 x1874670011679744/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828430.581036:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011679744/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828430.581042:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828430.581044:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828430.581046:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011679744:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828430.581050:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011679872:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828430.581069:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828430.581076:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828430/real 1787828430] req@ffff907d90844780 x1874670011679872/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828430.581084:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011679872/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828430.581090:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828430.581093:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828430.581095:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011679872:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828430.581099:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011680000:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828430.581109:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828430.581115:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828430/real 1787828430] req@ffff907d908452c0 x1874670011680000/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828430.581122:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011680000/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828430.581138:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828430.581140:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828430.581143:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011680000:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828430.708749:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828430.708829:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828431.732704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828431.732742:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828432.756702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828432.756723:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828433.780741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828433.780911:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828434.804757:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828434.804797:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828435.700755:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828435.700784:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828435.700789:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828435.700801:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3839 00000100:00080000:0.0:1787828435.700814:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828435.700824:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828435.700868:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011680128/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828435.700895:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828435.700898:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828435.700900:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828435.700903:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3839 00000100:00080000:0.0:1787828435.700907:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828435.700910:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828435.700919:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828435.700929:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011680256/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828435.700938:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828435.700941:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828435.700942:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828435.700947:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3839 00000100:00080000:0.0:1787828435.700949:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828435.700953:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828435.700957:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828435.700966:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011680128:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828435.700967:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011680384/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828435.700983:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828435.700985:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828435.700987:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828435.700989:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3839 00000100:00080000:0.0:1787828435.700992:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828435.700995:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828435.700998:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828435.701009:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011680512/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828435.701043:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828435.701092:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828435/real 1787828435] req@ffff907d90844000 x1874670011680128/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828435.701110:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011680128/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828435.701119:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828435.701123:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828435.701126:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011680128:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828435.701133:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011680256:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828435.701155:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828435.701165:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828435/real 1787828435] req@ffff907d908443c0 x1874670011680256/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828435.701175:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011680256/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828435.701182:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828435.701185:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828435.701188:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011680256:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828435.701207:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011680384:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828435.701221:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828435.701229:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828435/real 1787828435] req@ffff907d90844b40 x1874670011680384/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828435.701238:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011680384/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828435.701244:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828435.701247:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828435.701250:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011680384:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828435.701254:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011680512:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828435.701265:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828435.701271:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828435/real 1787828435] req@ffff907d90845e00 x1874670011680512/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828435.701280:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011680512/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828435.701286:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828435.701288:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828435.701291:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011680512:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828435.828695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828435.828860:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828436.852754:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828436.852860:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828437.876745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828437.876913:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828438.900766:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828438.900850:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828439.924806:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828439.924961:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828440.820718:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828440.820733:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828440.820738:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828440.820751:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3844 00000100:00080000:0.0:1787828440.820764:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828440.820774:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828440.820821:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011680640/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828440.820865:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828440.820870:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828440.820873:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828440.820877:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3844 00000100:00080000:0.0:1787828440.820882:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828440.820886:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828440.820898:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828440.820912:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011680768/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828440.820924:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828440.820928:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828440.820931:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828440.820935:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3844 00000100:00080000:0.0:1787828440.820939:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828440.820943:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828440.820949:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828440.820961:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011680896/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828440.820975:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828440.820979:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828440.820982:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828440.820993:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011680640:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828440.820995:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3844 00000100:00080000:0.0:1787828440.821010:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828440.821021:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828440.821031:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828440.821059:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011681024/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828440.821092:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828440.821168:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828440/real 1787828440] req@ffff907d908470c0 x1874670011680640/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828440.821192:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011680640/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828440.821206:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828440.821211:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828440.821217:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011680640:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828440.821226:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011680768:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828440.821253:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828440.821268:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828440/real 1787828440] req@ffff907d90846580 x1874670011680768/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828440.821283:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011680768/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828440.821293:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828440.821298:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828440.821302:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011680768:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828440.821330:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011680896:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828440.821351:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828440.821363:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828440/real 1787828440] req@ffff907d90847c00 x1874670011680896/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828440.821377:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011680896/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828440.821388:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828440.821391:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828440.821395:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011680896:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828440.821402:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011681024:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828440.821459:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828440.821482:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828440/real 1787828440] req@ffff907d90845e00 x1874670011681024/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828440.821505:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011681024/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828440.821521:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828440.821527:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828440.821534:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011681024:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828440.948733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828440.948825:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828441.972755:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828441.972812:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828442.996728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828442.996787:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828444.020694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828444.020738:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828445.044689:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828445.044815:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828445.940711:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828445.940725:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828445.940760:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828445.940776:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3849 00000100:00080000:0.0:1787828445.940792:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828445.940805:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828445.940862:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011681152/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828445.940919:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828445.940927:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828445.940931:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828445.940939:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3849 00000100:00080000:0.0:1787828445.940947:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828445.940955:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828445.940970:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828445.940994:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011681280/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828445.941009:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828445.941016:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828445.941029:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828445.941034:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3849 00000100:00100000:2.0:1787828445.941043:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011681152:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828445.941044:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828445.941057:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828445.941067:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828445.941083:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011681408/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828445.941096:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828445.941101:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828445.941104:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828445.941121:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3849 00000100:00080000:0.0:1787828445.941130:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828445.941137:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828445.941146:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828445.941169:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011681536/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828445.941228:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011681280:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828445.941296:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011681408:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828445.941452:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011681536:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828446.068688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828446.068713:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828447.092682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828447.092744:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828448.116733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828448.116834:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787828449.012985:0:2338:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:3.0:1787828449.013076:0:2338:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787828449.013099:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787828449.013151:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787828449.013184:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787828449.013196:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:2.0:1787828449.013274:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828445/real 1787828449] req@ffff907d90844000 x1874670011681408/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828449.013317:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011681408/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828449.013333:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828449.013339:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828449.013345:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011681408:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828449.013380:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828445/real 1787828449] req@ffff907d908452c0 x1874670011681536/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828449.013398:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011681536/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828449.013410:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828449.013416:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828449.013420:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011681536:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828449.013430:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828445/real 1787828449] req@ffff907d90844b40 x1874670011681152/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828449.013447:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011681152/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828449.013457:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828449.013461:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828449.013481:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011681152:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828449.013491:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828445/real 1787828449] req@ffff907d908443c0 x1874670011681280/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828449.013524:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011681280/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828449.013536:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828449.013540:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828449.013544:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011681280:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828449.140691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828449.140796:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828450.164723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828450.164813:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828451.060743:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828451.060754:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828451.060759:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828451.060787:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3854 00000100:00080000:0.0:1787828451.060800:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828451.060809:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828451.060874:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011681664/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828451.060911:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828451.060916:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828451.060919:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828451.060924:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3854 00000100:00080000:0.0:1787828451.060928:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828451.060932:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828451.060961:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828451.060974:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011681792/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828451.060987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828451.060992:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828451.060994:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828451.060998:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3854 00000100:00080000:0.0:1787828451.061001:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828451.061005:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828451.061012:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828451.061030:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011681664:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828451.061033:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011681920/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828451.061062:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828451.061067:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828451.061071:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828451.061078:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3854 00000100:00080000:0.0:1787828451.061084:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828451.061090:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828451.061098:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828451.061119:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011682048/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828451.061167:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828451.061232:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828451/real 1787828451] req@ffff907d90844780 x1874670011681664/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828451.061273:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011681664/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828451.061291:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828451.061298:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828451.061305:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011681664:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828451.061316:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011681792:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828451.061383:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828451.061408:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828451/real 1787828451] req@ffff907d90845680 x1874670011681792/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828451.061430:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011681792/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828451.061443:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828451.061449:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828451.061455:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011681792:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828451.061489:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011681920:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828451.061525:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828451.061547:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828451/real 1787828451] req@ffff907d90846d00 x1874670011681920/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828451.061568:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011681920/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828451.061591:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828451.061597:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828451.061603:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011681920:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828451.061664:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011682048:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828451.061702:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828451.061724:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828451/real 1787828451] req@ffff907d90846940 x1874670011682048/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828451.061744:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011682048/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828451.061759:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828451.061765:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828451.061770:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011682048:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828451.188697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828451.188839:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828452.212721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828452.212821:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828453.236698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828453.236748:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828454.260733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828454.260833:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828455.284671:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828455.284772:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828456.180771:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828456.180784:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828456.180791:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828456.180805:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3859 00000100:00080000:3.0:1787828456.180819:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828456.180829:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828456.180872:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011682176/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828456.180913:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828456.180919:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828456.180921:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828456.180926:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3859 00000100:00080000:3.0:1787828456.180942:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828456.180946:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828456.180958:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828456.180973:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011682304/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828456.180987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828456.180992:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828456.180994:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828456.181000:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3859 00000100:00080000:3.0:1787828456.181003:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828456.181007:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828456.181017:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828456.181019:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011682176:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828456.181034:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011682432/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828456.181051:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828456.181055:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828456.181057:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828456.181061:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3859 00000100:00080000:3.0:1787828456.181066:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828456.181071:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828456.181077:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828456.181092:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011682560/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828456.181111:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828456.181168:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828456/real 1787828456] req@ffff907da562da40 x1874670011682176/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828456.181191:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011682176/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828456.181206:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828456.181211:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828456.181225:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011682176:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828456.181234:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011682304:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828456.181259:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828456.181274:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828456/real 1787828456] req@ffff907da562f840 x1874670011682304/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828456.181288:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011682304/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828456.181326:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828456.181332:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828456.181348:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011682304:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828456.181383:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011682432:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828456.181412:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828456.181426:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828456/real 1787828456] req@ffff907da562e1c0 x1874670011682432/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828456.181441:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011682432/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828456.181452:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828456.181456:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828456.181460:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011682432:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828456.181468:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011682560:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828456.181487:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828456.181498:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828456/real 1787828456] req@ffff907da562cb40 x1874670011682560/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828456.181511:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011682560/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828456.181521:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828456.181525:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828456.181528:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011682560:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828456.308732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828456.308863:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828457.332746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828457.332798:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828458.356762:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828458.356800:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828459.380742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828459.380800:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828460.404687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828460.404712:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828461.300715:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828461.300726:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828461.300730:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828461.300741:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3865 00000100:00080000:3.0:1787828461.300752:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828461.300760:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828461.300799:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011682688/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828461.300829:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828461.300833:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828461.300836:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828461.300841:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3865 00000100:00080000:3.0:1787828461.300846:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828461.300850:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828461.300859:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828461.300875:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011682816/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828461.300888:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828461.300892:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828461.300894:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828461.300899:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3865 00000100:00100000:2.0:1787828461.300904:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011682688:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828461.300905:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828461.300915:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828461.300922:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828461.300936:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011682944/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828461.300958:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828461.300962:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1787828461.300966:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787828461.300967:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828461.300974:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3865 00000100:00080000:3.0:1787828461.300980:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828461.300985:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828461.300991:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828461.301006:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828461/real 1787828461] req@ffff907da562de00 x1874670011682688/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828461.301010:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011683072/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828461.301030:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011682688/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828461.301040:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828461.301043:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828461.301046:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011682688:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828461.301054:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011682816:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828461.301068:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828461.301076:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828461/real 1787828461] req@ffff907da562cb40 x1874670011682816/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828461.301084:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011682816/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828461.301089:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828461.301091:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828461.301094:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011682816:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828461.301111:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011682944:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828461.301122:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828461.301128:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828461/real 1787828461] req@ffff907da562e1c0 x1874670011682944/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828461.301135:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011682944/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828461.301141:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828461.301143:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828461.301145:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011682944:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828461.301149:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011683072:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828461.301157:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828461.301163:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828461/real 1787828461] req@ffff907da562f840 x1874670011683072/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828461.301170:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011683072/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828461.301175:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828461.301177:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828461.301179:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011683072:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828461.428695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828461.428760:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828462.452690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828462.452720:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828463.476733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828463.476852:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828464.500710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828464.500985:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828465.524737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828465.524793:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828466.420870:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828466.420883:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828466.420891:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828466.420921:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3870 00000100:00080000:1.0:1787828466.420935:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828466.420941:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828466.420977:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011683200/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828466.421009:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828466.421012:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828466.421013:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828466.421016:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3870 00000100:00080000:1.0:1787828466.421019:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828466.421021:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828466.421029:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828466.421039:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011683328/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828466.421048:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011683200:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828466.421055:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828466.421057:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828466.421058:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828466.421061:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3870 00000100:00080000:1.0:1787828466.421063:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828466.421074:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828466.421078:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828466.421086:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011683456/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828466.421092:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828466.421095:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828466.421096:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828466.421099:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3870 00000100:00080000:1.0:1787828466.421101:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828466.421103:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828466.421106:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828466.421114:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011683584/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828466.421151:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828466.421182:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828466/real 1787828466] req@ffff907d908a21c0 x1874670011683200/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828466.421195:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011683200/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828466.421203:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828466.421205:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828466.421208:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011683200:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828466.421226:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011683328:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828466.421269:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828466.421279:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828466/real 1787828466] req@ffff907d908a0f00 x1874670011683328/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828466.421287:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011683328/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828466.421293:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828466.421295:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828466.421298:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011683328:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828466.421302:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011683456:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828466.421313:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828466.421319:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828466/real 1787828466] req@ffff907d908a12c0 x1874670011683456/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828466.421327:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011683456/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828466.421333:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828466.421335:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828466.421337:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011683456:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828466.421340:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011683584:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828466.421349:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828466.421354:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828466/real 1787828466] req@ffff907d908a3840 x1874670011683584/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828466.421361:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011683584/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828466.421370:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828466.421373:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828466.421375:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011683584:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828466.548678:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828466.548739:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828467.572706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828467.572728:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828468.596676:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828468.596756:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828469.620712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828469.620794:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828470.644677:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828470.644731:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828471.540791:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828471.540804:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828471.540809:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828471.540823:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3875 00000100:00080000:0.0:1787828471.540839:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828471.540850:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828471.540904:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011683712/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828471.540950:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828471.540954:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828471.540957:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828471.540962:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3875 00000100:00080000:0.0:1787828471.540967:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828471.540974:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828471.540987:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828471.541011:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011683840/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828471.541011:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011683712:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828471.541034:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828471.541038:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828471.541040:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828471.541045:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3875 00000100:00080000:0.0:1787828471.541049:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828471.541061:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828471.541069:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828471.541082:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011683968/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828471.541095:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787828471.541102:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787828471.541104:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828471.541110:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828471.541118:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3875 00000100:00080000:0.0:1787828471.541126:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828471.541133:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828471.541141:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828471.541157:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828471/real 1787828471] req@ffff907d90847480 x1874670011683712/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828471.541169:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011684096/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828471.541190:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011683712/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828471.541246:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828471.541251:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828471.541259:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011683712:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828471.541287:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011683840:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828471.541327:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828471.541341:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828471/real 1787828471] req@ffff907d90845a40 x1874670011683840/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828471.541358:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011683840/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828471.541374:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828471.541379:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828471.541385:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011683840:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828471.541396:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011683968:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828471.541449:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828471.541472:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828471/real 1787828471] req@ffff907d90844f00 x1874670011683968/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828471.541510:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011683968/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828471.541520:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828471.541523:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828471.541526:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011683968:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828471.541533:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011684096:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828471.541556:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828471.541568:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828471/real 1787828471] req@ffff907d90846940 x1874670011684096/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828471.541580:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011684096/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828471.541588:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828471.541591:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828471.541595:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011684096:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828471.668740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828471.668883:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828472.692747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828472.692811:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828473.716739:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828473.716745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787828474.740783:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828474.740833:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828475.764708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828475.764764:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828476.660749:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828476.660758:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828476.660763:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828476.660771:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3880 00000100:00080000:1.0:1787828476.660780:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828476.660786:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828476.660819:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011684224/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828476.660846:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828476.660851:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828476.660852:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828476.660857:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3880 00000100:00080000:1.0:1787828476.660860:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828476.660866:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828476.660876:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828476.660890:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011684224:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828476.660896:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011684352/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828476.660910:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828476.660915:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828476.660917:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828476.660920:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3880 00000100:00080000:1.0:1787828476.660924:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828476.660927:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828476.660931:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828476.660941:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011684480/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828476.660949:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787828476.660958:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828476.660961:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828476.660962:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828476.660966:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3880 00000100:00080000:1.0:1787828476.660969:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828476.660971:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828476.660974:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828476.660982:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011684608/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828476.660989:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828476/real 1787828476] req@ffff907d908a1680 x1874670011684224/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828476.661004:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011684224/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828476.661013:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828476.661016:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828476.661018:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011684224:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828476.661037:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011684352:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828476.661052:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828476.661060:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828476/real 1787828476] req@ffff907d908a3840 x1874670011684352/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828476.661068:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011684352/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828476.661073:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828476.661075:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828476.661078:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011684352:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828476.661082:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011684480:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828476.661093:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828476.661098:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828476/real 1787828476] req@ffff907d908a12c0 x1874670011684480/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828476.661105:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011684480/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828476.661111:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828476.661112:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828476.661114:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011684480:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828476.661117:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011684608:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828476.661126:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828476.661132:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828476/real 1787828476] req@ffff907d908a0f00 x1874670011684608/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828476.661139:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011684608/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828476.661144:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828476.661146:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828476.661148:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011684608:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828476.788737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828476.788822:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828477.812759:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828477.812786:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828478.836741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828478.836802:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828479.860730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828479.860819:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828480.884760:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828480.884804:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828481.780718:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828481.780754:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828481.780758:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828481.780766:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3885 00000100:00080000:0.0:1787828481.780775:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828481.780783:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828481.780832:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011684736/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828481.780857:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828481.780860:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828481.780862:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828481.780865:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3885 00000100:00080000:0.0:1787828481.780868:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828481.780871:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828481.780879:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828481.780890:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011684864/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828481.780898:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828481.780902:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787828481.780902:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011684736:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828481.780904:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828481.780915:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3885 00000100:00080000:0.0:1787828481.780920:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828481.780924:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828481.780931:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828481.780970:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011684992/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828481.780980:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828481.780984:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828481.780986:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828481.780989:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3885 00000100:00080000:0.0:1787828481.780993:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828481.781014:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828481.781030:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787828481.781036:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828481.781044:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011685120/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828481.781103:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828481/real 1787828481] req@ffff907d90846d00 x1874670011684736/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828481.781196:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011684736/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828481.781205:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828481.781209:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828481.781212:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011684736:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828481.781231:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011684864:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787828481.781251:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828481.781262:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828481/real 1787828481] req@ffff907d90845680 x1874670011684864/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828481.781270:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011684864/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828481.781276:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828481.781278:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828481.781280:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011684864:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828481.781285:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011684992:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828481.781315:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828481.781324:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828481/real 1787828481] req@ffff907d90844780 x1874670011684992/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828481.781332:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011684992/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828481.781338:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828481.781341:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828481.781343:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011684992:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828481.781347:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011685120:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828481.781360:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828481.781367:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828481/real 1787828481] req@ffff907d908443c0 x1874670011685120/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828481.781383:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011685120/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828481.781388:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828481.781390:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828481.781393:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011685120:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828481.908746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828481.908788:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828482.932697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828482.932752:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828483.956694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828483.956793:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828484.980696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828484.980764:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828486.004723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828486.004901:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828486.900766:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828486.900777:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828486.900782:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828486.900793:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3890 00000100:00080000:1.0:1787828486.900804:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828486.900811:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828486.900850:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011685248/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828486.900880:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828486.900884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828486.900887:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828486.900892:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3890 00000100:00080000:1.0:1787828486.900897:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828486.900901:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828486.900922:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011685248:192.168.203.104@tcp:250:kworker.0 00000008:00100000:1.0:1787828486.900930:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828486.900949:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011685376/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828486.900970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828486.900975:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828486.900977:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787828486.900982:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787828486.900984:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3890 00000100:00080000:1.0:1787828486.900991:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828486.900997:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828486.901004:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828486.901020:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011685504/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828486.901022:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828486/real 1787828486] req@ffff907d908a21c0 x1874670011685248/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828486.901057:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828486.901062:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011685248/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828486.901066:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828486.901072:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828486.901079:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828486.901083:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3890 00000100:00080000:2.0:1787828486.901102:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828486.901108:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011685248:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828486.901111:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828486.901134:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828486.901140:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828486.901153:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011685376:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828486.901155:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011685632/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828486.901177:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828486.901209:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828486/real 1787828486] req@ffff907d908a2580 x1874670011685376/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828486.901218:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011685376/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828486.901231:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828486.901234:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828486.901237:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011685376:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828486.901242:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011685504:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828486.901256:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828486.901264:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828486/real 1787828486] req@ffff907d908a2940 x1874670011685504/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828486.901272:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011685504/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828486.901278:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828486.901280:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828486.901282:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011685504:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828486.901291:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011685632:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828486.901301:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828486.901307:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828486/real 1787828486] req@ffff907d908a0000 x1874670011685632/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828486.901315:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011685632/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828486.901320:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828486.901322:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828486.901324:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011685632:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828487.028694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828487.028793:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828488.052723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828488.052773:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828489.076703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828489.076742:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828490.100701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828490.100822:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828491.124706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828491.124773:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828492.020752:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828492.020766:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828492.020772:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828492.020786:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3895 00000100:00080000:0.0:1787828492.020798:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828492.020806:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828492.020857:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011685760/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828492.020889:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828492.020904:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828492.020907:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828492.020911:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3895 00000100:00080000:0.0:1787828492.020915:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828492.020918:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828492.020929:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828492.020940:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011685888/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828492.020951:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828492.020955:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828492.020957:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828492.020961:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3895 00000100:00080000:0.0:1787828492.020965:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828492.020968:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828492.020973:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828492.020982:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011686016/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828492.020995:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011685760:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828492.020996:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828492.021011:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828492.021013:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828492.021018:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3895 00000100:00080000:0.0:1787828492.021021:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828492.021024:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828492.021029:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828492.021043:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011686144/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828492.021140:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828492.021211:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828492/real 1787828492] req@ffff907d90844b40 x1874670011685760/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828492.021231:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011685760/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828492.021242:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828492.021246:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828492.021250:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011685760:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828492.021258:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011685888:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828492.021281:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828492.021310:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828492/real 1787828492] req@ffff907d908452c0 x1874670011685888/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828492.021334:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011685888/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828492.021343:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828492.021346:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828492.021350:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011685888:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828492.021379:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011686016:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828492.021446:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828492.021464:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828492/real 1787828492] req@ffff907d90844000 x1874670011686016/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828492.021485:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011686016/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828492.021498:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828492.021503:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828492.021508:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011686016:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828492.021515:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011686144:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828492.021537:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828492.021548:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828492/real 1787828492] req@ffff907d90845e00 x1874670011686144/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828492.021560:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011686144/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828492.021568:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828492.021571:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828492.021574:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011686144:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828492.148719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828492.148844:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828493.172736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828493.172748:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828494.196703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828494.196765:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828495.220684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828495.220708:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828496.244736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828496.244817:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828497.140699:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828497.140711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828497.140717:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828497.140731:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3900 00000100:00080000:1.0:1787828497.140743:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828497.140751:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828497.140788:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011686272/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828497.140821:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828497.140837:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828497.140839:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828497.140843:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3900 00000100:00080000:1.0:1787828497.140846:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828497.140850:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828497.140867:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828497.140882:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011686400/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828497.140893:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011686272:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828497.140900:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828497.140906:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828497.140908:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828497.140913:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3900 00000100:00080000:1.0:1787828497.140916:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828497.140920:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828497.140925:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828497.140942:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011686528/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828497.140952:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828497.140957:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828497.140959:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828497.140962:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3900 00000100:00080000:1.0:1787828497.140966:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787828497.140974:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787828497.140975:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828497.140986:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828497.141000:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011686656/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828497.141050:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828497/real 1787828497] req@ffff907d908a1e00 x1874670011686272/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828497.141094:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011686272/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828497.141109:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828497.141114:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828497.141119:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011686272:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828497.141149:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011686400:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828497.141181:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828497.141198:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828497/real 1787828497] req@ffff907d908a2d00 x1874670011686400/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828497.141214:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011686400/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828497.141225:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828497.141229:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828497.141233:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011686400:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828497.141241:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011686528:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828497.141279:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828497.141290:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828497/real 1787828497] req@ffff907d908a0000 x1874670011686528/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828497.141309:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011686528/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828497.141318:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828497.141320:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828497.141323:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011686528:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828497.141347:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011686656:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828497.141369:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828497.141381:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828497/real 1787828497] req@ffff907d908a2940 x1874670011686656/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828497.141412:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011686656/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828497.141422:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828497.141425:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828497.141428:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011686656:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828497.268705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828497.268879:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828498.292759:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828498.292856:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828499.316679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828499.316774:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828500.340713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828500.340792:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828501.364705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828501.364774:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828502.260657:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828502.260666:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828502.260670:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828502.260678:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3906 00000100:00080000:1.0:1787828502.260688:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828502.260693:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828502.260738:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011686784/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828502.260765:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828502.260768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828502.260770:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828502.260774:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3906 00000100:00080000:1.0:1787828502.260778:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828502.260790:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828502.260800:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828502.260811:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011686784:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828502.260816:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011686912/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828502.260826:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828502.260829:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828502.260831:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828502.260834:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3906 00000100:00080000:1.0:1787828502.260847:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828502.260853:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828502.260869:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787828502.260886:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828502.260887:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011687040/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828502.260902:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828502.260906:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828502.260908:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828502.260941:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3906 00000100:00080000:1.0:1787828502.260949:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828502.260957:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828502.260967:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828502.260974:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828502/real 1787828502] req@ffff907d908a2580 x1874670011686784/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828502.261053:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011687168/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828502.261054:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011686784/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828502.261090:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828502.261094:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828502.261098:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011686784:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828502.261152:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011686912:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828502.261182:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828502.261196:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828502/real 1787828502] req@ffff907d908a21c0 x1874670011686912/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828502.261209:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011686912/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828502.261219:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828502.261222:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828502.261226:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011686912:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828502.261232:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011687040:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828502.261249:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828502.261259:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828502/real 1787828502] req@ffff907d908a0f00 x1874670011687040/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828502.261271:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011687040/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828502.261286:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828502.261289:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828502.261292:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011687040:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828502.261298:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011687168:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828502.261312:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828502.261321:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828502/real 1787828502] req@ffff907d908a12c0 x1874670011687168/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828502.261333:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011687168/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828502.261346:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828502.261352:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828502.261357:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011687168:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828502.388695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828502.388743:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828503.412723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828503.412771:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828504.436813:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828504.436924:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828505.460730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828505.460842:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828506.484718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828506.484837:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828507.380786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828507.380794:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828507.380798:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828507.380806:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3911 00000100:00080000:3.0:1787828507.380816:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828507.380821:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828507.380853:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011687296/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828507.380875:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828507.380878:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828507.380879:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828507.380882:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3911 00000100:00080000:3.0:1787828507.380884:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828507.380888:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828507.380895:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828507.380905:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011687424/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828507.380913:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828507.380917:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011687296:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828507.380917:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828507.380920:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828507.380928:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3911 00000100:00080000:3.0:1787828507.380930:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828507.380932:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828507.380935:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828507.380959:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011687552/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828507.380976:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828507.380979:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828507.380981:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828507.380983:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3911 00000100:00080000:3.0:1787828507.380992:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828507.380995:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828507.380999:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787828507.381003:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828507.381012:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011687680/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828507.381052:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828507/real 1787828507] req@ffff907da562da40 x1874670011687296/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828507.381072:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011687296/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828507.381080:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828507.381083:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828507.381086:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011687296:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828507.381105:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011687424:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828507.381123:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828507.381133:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828507/real 1787828507] req@ffff907da562f0c0 x1874670011687424/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828507.381141:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011687424/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828507.381147:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828507.381149:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828507.381151:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011687424:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828507.381155:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011687552:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828507.381166:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828507.381172:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828507/real 1787828507] req@ffff907da562c3c0 x1874670011687552/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828507.381180:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011687552/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828507.381189:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828507.381191:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828507.381193:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011687552:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828507.381197:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011687680:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828507.381206:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828507.381212:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828507/real 1787828507] req@ffff907da562e940 x1874670011687680/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828507.381219:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011687680/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828507.381224:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828507.381226:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828507.381228:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011687680:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828507.508696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828507.508821:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828508.532782:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828508.532837:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828509.556730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828509.556793:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828510.580726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828510.580762:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828511.604712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828511.604760:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828512.500683:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828512.500693:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828512.500698:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828512.500708:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3916 00000100:00080000:3.0:1787828512.500720:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828512.500728:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828512.500768:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011687808/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828512.500801:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828512.500805:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828512.500807:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828512.500811:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3916 00000100:00080000:3.0:1787828512.500814:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828512.500818:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828512.500828:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828512.500839:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011687936/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828512.500850:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828512.500854:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828512.500856:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828512.500859:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3916 00000100:00080000:3.0:1787828512.500861:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828512.500865:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828512.500870:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828512.500879:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011688064/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828512.500889:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828512.500893:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828512.500895:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828512.500898:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3916 00000100:00080000:3.0:1787828512.500902:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828512.500906:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828512.500933:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828512.500938:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011687808:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828512.500951:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011688192/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828512.501053:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011687936:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828512.501103:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011688064:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828512.501140:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011688192:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828512.628826:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828512.628875:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828513.652674:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828513.652739:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828514.676711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828514.676823:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787828515.572938:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:0.0:1787828515.572989:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787828515.573005:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787828515.573041:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787828515.573053:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787828515.573065:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:2.0:1787828515.573094:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828512/real 1787828515] req@ffff907da562c000 x1874670011688192/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828515.573124:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011688192/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828515.573141:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828515.573148:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828515.573154:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011688192:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828515.573165:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828512/real 1787828515] req@ffff907da562fc00 x1874670011687808/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828515.573185:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011687808/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828515.573197:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828515.573202:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828515.573207:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011687808:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828515.573217:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828512/real 1787828515] req@ffff907da562d2c0 x1874670011687936/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828515.573235:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011687936/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828515.573256:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828515.573261:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828515.573265:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011687936:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828515.573274:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828512/real 1787828515] req@ffff907da562e580 x1874670011688064/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828515.573290:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011688064/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828515.573302:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828515.573306:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828515.573310:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011688064:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828515.700705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828515.700733:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828516.724756:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828516.724902:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828517.620694:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828517.620708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828517.620714:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828517.620728:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3921 00000100:00080000:3.0:1787828517.620744:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828517.620753:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828517.620807:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011688320/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828517.620840:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828517.620854:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828517.620857:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828517.620865:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3921 00000100:00080000:3.0:1787828517.620869:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828517.620888:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828517.620905:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828517.620953:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011688448/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828517.620984:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828517.620991:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828517.620995:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828517.621004:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3921 00000100:00100000:2.0:1787828517.621019:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011688320:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828517.621019:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828517.621037:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828517.621052:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828517.621094:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011688576/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828517.621121:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828517.621127:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828517.621131:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828517.621144:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3921 00000800:00000100:2.0:1787828517.621156:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787828517.621188:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828517.621204:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828517.621218:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828517.621247:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011688704/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828517.621280:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828517/real 1787828517] req@ffff907da562f480 x1874670011688320/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828517.621316:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011688320/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828517.621333:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828517.621340:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828517.621353:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011688320:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828517.621367:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011688448:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828517.621405:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828517.621426:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828517/real 1787828517] req@ffff907da562c780 x1874670011688448/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828517.621453:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011688448/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828517.621466:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828517.621471:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828517.621475:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011688448:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828517.621513:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011688576:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828517.621547:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828517.621566:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828517/real 1787828517] req@ffff907da562ed00 x1874670011688576/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828517.621583:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011688576/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828517.621595:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828517.621600:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828517.621652:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011688576:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828517.621665:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011688704:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828517.621694:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828517.621712:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828517/real 1787828517] req@ffff907da562cf00 x1874670011688704/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828517.621732:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011688704/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828517.621745:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828517.621749:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828517.621754:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011688704:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828517.748746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828517.748858:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828518.772728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828518.772787:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828519.796778:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828519.796929:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828520.820695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828520.820782:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828521.844804:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828521.844921:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828522.740763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828522.740779:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828522.740786:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828522.740803:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3926 00000100:00080000:3.0:1787828522.740820:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828522.740831:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828522.740926:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011688832/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828522.740969:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828522.740975:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828522.740977:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828522.740983:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3926 00000100:00080000:3.0:1787828522.740988:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828522.740993:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828522.741009:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828522.741029:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011688960/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828522.741044:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828522.741048:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828522.741051:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828522.741056:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3926 00000100:00080000:3.0:1787828522.741060:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828522.741065:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828522.741072:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828522.741091:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011689088/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828522.741093:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011688832:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828522.741135:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828522.741140:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828522.741142:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828522.741150:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3926 00000100:00080000:3.0:1787828522.741157:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828522.741164:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828522.741174:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828522.741199:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011689216/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828522.741203:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828522.741268:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828522/real 1787828522] req@ffff907da562d680 x1874670011688832/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828522.741291:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011688832/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828522.741304:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828522.741309:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828522.741314:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011688832:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828522.741323:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011688960:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828522.741345:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828522.741357:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828522/real 1787828522] req@ffff907da562cf00 x1874670011688960/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828522.741370:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011688960/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828522.741378:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828522.741382:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828522.741385:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011688960:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828522.741478:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011689088:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828522.741500:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828522.741512:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828522/real 1787828522] req@ffff907da562ed00 x1874670011689088/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828522.741524:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011689088/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828522.741533:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828522.741537:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828522.741540:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011689088:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828522.741546:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011689216:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828522.741567:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828522.741583:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828522/real 1787828522] req@ffff907da562c780 x1874670011689216/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828522.741602:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011689216/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828522.741679:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828522.741683:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828522.741687:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011689216:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828522.868696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828522.868862:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828523.892745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828523.892920:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828524.916746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828524.916861:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828525.940800:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828525.940801:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828526.964673:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828526.964787:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828527.860850:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828527.860885:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828527.860894:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828527.860912:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3931 00000100:00080000:1.0:1787828527.860930:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828527.860942:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828527.861017:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011689344/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828527.861070:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828527.861075:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828527.861078:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828527.861084:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3931 00000100:00080000:1.0:1787828527.861090:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828527.861096:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828527.861112:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828527.861133:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011689472/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828527.861148:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828527.861153:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828527.861155:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828527.861161:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3931 00000100:00080000:1.0:1787828527.861165:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828527.861170:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828527.861176:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828527.861186:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011689344:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828527.861217:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011689600/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828527.861257:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828527.861264:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828527.861268:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828527.861277:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3931 00000100:00080000:1.0:1787828527.861286:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787828527.861290:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787828527.861300:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828527.861314:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828527.861354:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828527/real 1787828527] req@ffff907d908a3840 x1874670011689344/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828527.861355:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011689728/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828527.861454:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011689344/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828527.861467:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828527.861472:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828527.861475:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011689344:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828527.861482:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011689472:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828527.861504:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828527.861536:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828527/real 1787828527] req@ffff907d908a1680 x1874670011689472/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828527.861556:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011689472/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828527.861565:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828527.861568:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828527.861571:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011689472:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828527.861619:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011689600:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828527.861733:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828527.861759:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828527/real 1787828527] req@ffff907d908a12c0 x1874670011689600/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828527.861776:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011689600/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828527.861787:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828527.861791:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828527.861795:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011689600:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828527.861833:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011689728:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828527.861880:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828527.861893:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828527/real 1787828527] req@ffff907d908a0f00 x1874670011689728/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828527.861904:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011689728/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828527.861920:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828527.861923:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828527.861926:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011689728:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828527.988731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828527.988832:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828529.012754:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828529.012889:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828530.036737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828530.037021:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828531.060753:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828531.060831:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828532.084757:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828532.084882:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828532.980753:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828532.980792:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828532.980799:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828532.980827:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3936 00000100:00080000:2.0:1787828532.980839:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828532.980847:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828532.980887:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011689856/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828532.980916:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828532.980920:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828532.980922:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828532.980925:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3936 00000100:00080000:2.0:1787828532.980929:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828532.980932:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828532.980942:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828532.980955:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011689984/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828532.980967:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828532.980973:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787828532.980974:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011689856:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828532.980975:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828532.980985:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3936 00000100:00080000:2.0:1787828532.980992:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828532.980995:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828532.981000:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828532.981012:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011690112/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828532.981021:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828532.981025:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828532.981027:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828532.981029:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3936 00000100:00080000:2.0:1787828532.981032:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828532.981035:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828532.981040:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828532.981049:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011690240/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828532.981072:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828532.981129:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828532/real 1787828532] req@ffff907d9114bc00 x1874670011689856/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828532.981155:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011689856/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828532.981171:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828532.981177:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828532.981183:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011689856:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828532.981246:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011689984:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787828532.981287:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828532.981310:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828532/real 1787828532] req@ffff907d91149680 x1874670011689984/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828532.981331:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011689984/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828532.981363:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828532.981369:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828532.981375:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011689984:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828532.981385:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011690112:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828532.981423:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828532.981444:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828532/real 1787828532] req@ffff907d9114b480 x1874670011690112/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828532.981464:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011690112/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828532.981478:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828532.981483:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828532.981489:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011690112:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828532.981498:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011690240:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828532.981529:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828532.981548:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828532/real 1787828532] req@ffff907d911483c0 x1874670011690240/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828532.981567:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011690240/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828532.981579:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828532.981583:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828532.981588:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011690240:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828533.108760:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828533.108832:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828534.132804:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828534.132907:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828535.156736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828535.156828:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828536.180732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828536.180862:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828537.204758:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828537.204882:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828538.100813:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828538.100829:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828538.100835:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828538.100851:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3941 00000100:00080000:0.0:1787828538.100868:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828538.100878:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828538.100935:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011690368/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828538.100977:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828538.100982:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828538.100988:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828538.100998:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3941 00000100:00080000:0.0:1787828538.101007:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828538.101016:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828538.101034:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828538.101040:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011690368:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828538.101068:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011690496/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828538.101095:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828538.101101:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828538.101104:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787828538.101112:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787828538.101113:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3941 00000100:00080000:0.0:1787828538.101125:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828538.101133:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828538.101142:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828538.101163:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011690624/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828538.101177:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828538/real 1787828538] req@ffff907d90847c00 x1874670011690368/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828538.101197:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828538.101205:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011690368/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828538.101207:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828538.101216:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828538.101239:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3941 00000100:00080000:0.0:1787828538.101249:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828538.101253:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828538.101256:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828538.101271:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1787828538.101275:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828538.101285:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011690368:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828538.101303:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011690752/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828538.101322:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011690496:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787828538.101357:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828538.101370:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828538/real 1787828538] req@ffff907d90846580 x1874670011690496/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828538.101379:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011690496/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828538.101385:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828538.101388:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828538.101390:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011690496:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828538.101395:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011690624:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828538.101417:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828538.101425:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828538/real 1787828538] req@ffff907d908470c0 x1874670011690624/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828538.101433:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011690624/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828538.101443:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828538.101446:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828538.101448:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011690624:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828538.101458:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011690752:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828538.101470:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828538.101476:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828538/real 1787828538] req@ffff907d90845e00 x1874670011690752/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828538.101484:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011690752/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828538.101489:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828538.101491:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828538.101493:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011690752:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828538.228703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828538.228820:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828539.252713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828539.252787:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828540.276675:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828540.276756:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828541.300701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828541.300802:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828542.324688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828542.324799:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828543.220757:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828543.220768:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828543.220773:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828543.220795:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3946 00000100:00080000:0.0:1787828543.220807:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828543.220815:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828543.220855:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011690880/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828543.220886:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828543.220890:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828543.220892:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828543.220896:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3946 00000100:00080000:0.0:1787828543.220900:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828543.220903:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828543.220914:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828543.220928:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011691008/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828543.220940:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828543.220944:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828543.220946:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828543.220950:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3946 00000100:00080000:0.0:1787828543.220954:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828543.220958:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828543.220963:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828543.220974:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011691136/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828543.220989:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828543.220995:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1787828543.220995:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011690880:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828543.221005:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828543.221028:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3946 00000100:00080000:0.0:1787828543.221033:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828543.221037:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828543.221043:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828543.221056:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011691264/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828543.221135:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828543.221204:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828543/real 1787828543] req@ffff907d90844000 x1874670011690880/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828543.221225:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011690880/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828543.221249:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828543.221255:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828543.221261:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011690880:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828543.221272:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011691008:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787828543.221309:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828543.221322:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828543/real 1787828543] req@ffff907d908452c0 x1874670011691008/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828543.221333:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011691008/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828543.221340:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828543.221343:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828543.221346:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011691008:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828543.221374:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011691136:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828543.221392:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828543.221401:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828543/real 1787828543] req@ffff907d90844b40 x1874670011691136/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828543.221410:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011691136/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828543.221428:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828543.221431:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828543.221434:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011691136:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828543.221441:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011691264:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828543.221456:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828543.221464:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828543/real 1787828543] req@ffff907d908443c0 x1874670011691264/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828543.221474:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011691264/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828543.221481:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828543.221484:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828543.221487:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011691264:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828543.348803:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828543.348917:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828544.372792:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828544.372922:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828545.396755:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828545.396879:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828546.420793:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828546.420883:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828547.444774:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828547.444840:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828548.340762:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828548.340775:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828548.340780:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828548.340793:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3952 00000100:00080000:1.0:1787828548.340807:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828548.340816:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828548.340878:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011691392/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828548.340905:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828548.340909:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828548.340912:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828548.340917:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3952 00000100:00080000:1.0:1787828548.340922:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828548.340929:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828548.340942:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828548.340953:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011691392:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828548.340965:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011691520/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828548.340983:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828548.340988:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828548.340991:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828548.340997:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3952 00000100:00080000:1.0:1787828548.341002:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828548.341006:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828548.341015:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787828548.341020:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828548.341039:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011691648/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828548.341055:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828548.341060:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1787828548.341066:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828548/real 1787828548] req@ffff907d908a21c0 x1874670011691392/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828548.341067:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828548.341081:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3952 00000100:00080000:1.0:1787828548.341090:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828548.341096:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011691392/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828548.341097:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828548.341108:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1787828548.341115:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828548.341121:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828548.341127:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011691776/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828548.341128:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011691392:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828548.341161:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011691520:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828548.341188:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828548.341205:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828548/real 1787828548] req@ffff907d908a2580 x1874670011691520/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828548.341223:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011691520/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828548.341245:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828548.341249:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828548.341252:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011691520:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828548.341257:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011691648:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828548.341278:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828548.341289:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828548/real 1787828548] req@ffff907d908a2940 x1874670011691648/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828548.341298:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011691648/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828548.341305:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828548.341308:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828548.341311:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011691648:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828548.341315:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011691776:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828548.341328:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828548.341335:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828548/real 1787828548] req@ffff907d908a0000 x1874670011691776/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828548.341344:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011691776/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828548.341350:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828548.341352:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828548.341356:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011691776:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828548.468740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828548.468822:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828549.492731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828549.492768:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828550.516753:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828550.516811:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828551.540795:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828551.540949:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828552.564740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828552.564925:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828553.460835:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828553.460848:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828553.460852:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828553.460861:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3957 00000100:00080000:1.0:1787828553.460872:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828553.460881:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828553.460913:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011691904/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828553.460943:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828553.460948:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828553.460950:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828553.460956:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3957 00000100:00080000:1.0:1787828553.460963:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828553.460970:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828553.460982:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828553.461001:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011692032/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828553.461011:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828553.461014:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828553.461018:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828553.461032:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3957 00000100:00080000:1.0:1787828553.461036:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828553.461040:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828553.461045:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828553.461054:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011692160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828553.461064:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828553.461066:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828553.461069:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828553.461080:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3957 00000100:00100000:3.0:1787828553.461083:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011691904:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828553.461084:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828553.461092:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828553.461098:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828553.461118:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011692288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828553.461771:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828553.461863:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828553/real 1787828553] req@ffff907d908a2d00 x1874670011691904/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828553.461890:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011691904/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828553.461913:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828553.461923:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828553.461932:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011691904:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828553.461949:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011692032:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828553.462000:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828553.462025:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828553/real 1787828553] req@ffff907d908a1e00 x1874670011692032/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828553.462042:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011692032/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828553.462053:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828553.462060:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828553.462067:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011692032:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828553.462116:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011692160:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828553.462173:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828553.462200:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828553/real 1787828553] req@ffff907d908a0000 x1874670011692160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828553.462216:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011692160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828553.462228:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828553.462232:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828553.462236:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011692160:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828553.462244:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011692288:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828553.462267:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828553.462289:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828553/real 1787828553] req@ffff907d908a2940 x1874670011692288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828553.462303:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011692288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828553.462317:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828553.462322:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828553.462328:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011692288:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828553.588755:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828553.588811:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828554.612822:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828554.612930:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828555.636798:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828555.636865:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828556.660713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828556.660835:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828557.684747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828557.684782:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828558.580776:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828558.580786:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828558.580790:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828558.580801:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3962 00000100:00080000:0.0:1787828558.580811:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828558.580818:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828558.580885:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011692416/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828558.580919:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828558.580940:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828558.580941:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828558.580945:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3962 00000100:00080000:0.0:1787828558.580949:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828558.580954:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828558.580963:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828558.580975:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011692416:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828558.580978:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011692544/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828558.580990:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828558.580993:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828558.580995:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828558.580998:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3962 00000100:00080000:0.0:1787828558.581001:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828558.581003:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828558.581007:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828558.581016:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011692672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828558.581023:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828558.581027:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828558.581028:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828558.581031:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3962 00000100:00080000:0.0:1787828558.581033:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828558.581041:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:3.0:1787828558.581041:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:0.0:1787828558.581049:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828558.581058:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011692800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828558.581086:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828558/real 1787828558] req@ffff907d90844780 x1874670011692416/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828558.581102:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011692416/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828558.581110:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828558.581114:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828558.581117:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011692416:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828558.581136:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011692544:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828558.581153:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828558.581162:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828558/real 1787828558] req@ffff907d90845680 x1874670011692544/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828558.581176:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011692544/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828558.581182:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828558.581185:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828558.581187:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011692544:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828558.581191:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011692672:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828558.581204:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828558.581210:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828558/real 1787828558] req@ffff907d90846d00 x1874670011692672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828558.581219:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011692672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828558.581224:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828558.581226:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828558.581228:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011692672:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828558.581232:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011692800:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828558.581248:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828558.581254:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828558/real 1787828558] req@ffff907d90846940 x1874670011692800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828558.581262:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011692800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828558.581273:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828558.581275:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828558.581277:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011692800:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828558.708728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828558.708782:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828559.732721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828559.732785:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828560.756787:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828560.756865:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828561.780718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828561.780773:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828562.804720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828562.804765:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828563.700701:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828563.700709:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828563.700712:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828563.700721:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3967 00000100:00080000:2.0:1787828563.700731:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828563.700736:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828563.700770:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011692928/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828563.700792:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828563.700795:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828563.700796:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828563.700799:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3967 00000100:00080000:2.0:1787828563.700803:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828563.700806:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828563.700815:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828563.700824:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011693056/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828563.700831:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011692928:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828563.700835:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828563.700839:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828563.700840:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828563.700843:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3967 00000100:00080000:2.0:1787828563.700845:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828563.700847:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828563.700851:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828563.700860:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011693184/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828563.700867:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828563.700869:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828563.700871:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828563.700873:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3967 00000100:00080000:2.0:1787828563.700883:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828563.700886:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828563.700889:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787828563.700891:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828563.700900:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011693312/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828563.700931:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828563/real 1787828563] req@ffff907d9114b840 x1874670011692928/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828563.700945:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011692928/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828563.700953:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828563.700956:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828563.700959:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011692928:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828563.700976:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011693056:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828563.700991:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828563.700999:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828563/real 1787828563] req@ffff907d91149a40 x1874670011693056/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828563.701007:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011693056/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828563.701012:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828563.701015:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828563.701017:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011693056:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828563.701021:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011693184:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828563.701032:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828563.701037:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828563/real 1787828563] req@ffff907d9114ad00 x1874670011693184/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828563.701045:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011693184/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828563.701066:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828563.701068:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828563.701070:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011693184:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828563.701074:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011693312:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828563.701085:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828563.701091:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828563/real 1787828563] req@ffff907d9114a580 x1874670011693312/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828563.701107:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011693312/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828563.701113:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828563.701116:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828563.701118:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011693312:192.168.203.104@tcp:8:kworker.0 00010000:00010000:0.0:1787828563.828762:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828563.828762:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787828564.852791:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828564.852917:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828565.876727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828565.876904:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828566.900723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828566.900803:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828567.924767:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828567.924866:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828568.820758:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828568.820775:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828568.820781:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828568.820799:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3972 00000100:00080000:0.0:1787828568.820818:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828568.820828:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828568.820892:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011693440/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828568.820935:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828568.820941:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828568.820944:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828568.820952:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3972 00000100:00080000:0.0:1787828568.820958:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828568.820966:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828568.820982:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828568.820992:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011693440:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828568.821004:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011693568/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828568.821016:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828568.821020:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828568.821022:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828568.821027:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3972 00000100:00080000:0.0:1787828568.821031:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828568.821035:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828568.821041:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828568.821051:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011693696/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828568.821060:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828568.821080:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828568.821081:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828568.821083:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3972 00000100:00080000:0.0:1787828568.821086:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828568.821089:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828568.821093:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828568.821102:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011693824/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828568.821107:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828568.821177:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828568/real 1787828568] req@ffff907d90844f00 x1874670011693440/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828568.821198:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011693440/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828568.821209:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828568.821213:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828568.821217:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011693440:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828568.821257:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011693568:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828568.821288:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828568.821321:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828568/real 1787828568] req@ffff907d90845a40 x1874670011693568/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828568.821354:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011693568/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828568.821361:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828568.821364:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828568.821367:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011693568:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828568.821372:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011693696:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828568.821391:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828568.821401:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828568/real 1787828568] req@ffff907d90847480 x1874670011693696/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828568.821441:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011693696/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828568.821449:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828568.821452:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828568.821455:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011693696:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828568.821460:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011693824:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828568.821478:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828568.821487:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828568/real 1787828568] req@ffff907d90846940 x1874670011693824/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828568.821497:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011693824/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828568.821504:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828568.821506:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828568.821509:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011693824:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828568.948754:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828568.948840:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828569.972695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828569.972831:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828570.996797:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828570.996870:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828572.020721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828572.020817:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828573.044704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828573.044798:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828573.940819:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828573.940831:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828573.940835:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828573.940857:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3977 00000100:00080000:1.0:1787828573.940869:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828573.940892:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828573.940929:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011693952/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828573.940977:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828573.940981:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828573.940984:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828573.940988:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3977 00000100:00080000:1.0:1787828573.940992:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828573.940999:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828573.941011:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828573.941025:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011694080/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828573.941049:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011693952:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828573.941084:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828573.941089:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828573.941091:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828573.941098:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3977 00000100:00080000:1.0:1787828573.941103:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828573.941110:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828573.941119:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828573.941154:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011694208/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828573.941174:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828573.941181:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828573.941185:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828573.941194:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3977 00000800:00000100:2.0:1787828573.941201:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787828573.941211:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828573.941225:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828573.941238:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828573.941278:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011694336/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828573.941308:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828573/real 1787828573] req@ffff907d908a2580 x1874670011693952/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828573.941355:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011693952/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828573.941372:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828573.941378:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828573.941384:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011693952:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828573.941433:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011694080:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828573.941505:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828573.941539:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828573/real 1787828573] req@ffff907d908a21c0 x1874670011694080/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828573.941549:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011694080/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828573.941555:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828573.941558:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828573.941560:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011694080:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828573.941565:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011694208:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828573.941599:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828573.941613:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828573/real 1787828573] req@ffff907d908a0f00 x1874670011694208/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828573.941627:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011694208/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828573.941686:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828573.941691:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828573.941695:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011694208:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828573.941703:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011694336:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828573.941730:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828573.941743:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828573/real 1787828573] req@ffff907d908a12c0 x1874670011694336/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828573.941758:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011694336/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828573.941768:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828573.941772:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828573.941776:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011694336:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828574.068700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828574.068775:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828575.092674:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828575.092778:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828576.116725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828576.116827:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828577.140734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828577.140851:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828578.164724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828578.164807:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828579.060718:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828579.060733:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828579.060740:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828579.060755:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3982 00000100:00080000:1.0:1787828579.060769:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828579.060777:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828579.060825:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011694464/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828579.060880:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828579.060887:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828579.060890:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828579.060895:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3982 00000100:00080000:1.0:1787828579.060901:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828579.060907:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828579.060922:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828579.060949:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011694592/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828579.060969:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828579.060976:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828579.060980:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828579.060987:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3982 00000100:00100000:2.0:1787828579.060995:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011694464:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828579.060997:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828579.061012:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828579.061023:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828579.061047:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011694720/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828579.061066:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828579.061072:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828579.061075:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828579.061083:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3982 00000100:00080000:1.0:1787828579.061090:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828579.061098:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828579.061107:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828579.061130:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011694592:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828579.061132:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011694848/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828579.061223:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011694720:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828579.061262:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011694848:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828579.188719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828579.188804:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828580.212734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828580.212887:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828581.236740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828581.236828:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787828582.133023:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787828582.133101:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787828582.133125:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828582.133178:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828582.133191:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828582.133203:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:2.0:1787828582.133253:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828579/real 1787828582] req@ffff907d908a12c0 x1874670011694720/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828582.133291:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011694720/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828582.133308:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828582.133313:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828582.133319:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011694720:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828582.133329:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828579/real 1787828582] req@ffff907d908a0f00 x1874670011694848/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828582.133343:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011694848/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828582.133353:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828582.133357:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828582.133361:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011694848:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828582.133369:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828579/real 1787828582] req@ffff907d908a1680 x1874670011694464/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828582.133383:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011694464/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828582.133392:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828582.133396:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828582.133400:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011694464:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828582.133408:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828579/real 1787828582] req@ffff907d908a3840 x1874670011694592/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828582.133422:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011694592/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828582.133446:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828582.133450:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828582.133454:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011694592:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828582.260772:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828582.260904:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828583.284703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828583.284789:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828584.180708:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828584.180723:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828584.180728:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828584.180737:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3987 00000100:00080000:2.0:1787828584.180748:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828584.180759:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828584.180792:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011694976/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828584.180817:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828584.180820:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828584.180822:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828584.180825:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3987 00000100:00080000:2.0:1787828584.180828:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828584.180832:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828584.180842:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828584.180859:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011694976:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828584.180862:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011695104/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828584.180879:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828584.180883:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828584.180884:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828584.180888:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3987 00000100:00080000:2.0:1787828584.180892:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828584.180895:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828584.180902:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828584.180912:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011695232/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828584.180921:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828584.180924:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828584.180926:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828584.180931:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3987 00000100:00080000:2.0:1787828584.180935:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787828584.180937:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787828584.180941:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828584.180947:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828584.180957:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011695360/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828584.181006:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828584/real 1787828584] req@ffff907d911492c0 x1874670011694976/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828584.181033:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011694976/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828584.181045:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828584.181048:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828584.181052:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011694976:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828584.181077:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011695104:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828584.181102:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828584.181115:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828584/real 1787828584] req@ffff907d91148f00 x1874670011695104/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828584.181125:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011695104/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828584.181132:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828584.181136:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828584.181141:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011695104:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828584.181149:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011695232:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828584.181175:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828584.181190:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828584/real 1787828584] req@ffff907d9114a940 x1874670011695232/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828584.181207:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011695232/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828584.181218:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828584.181222:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828584.181227:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011695232:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828584.181235:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011695360:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828584.181262:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828584.181277:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828584/real 1787828584] req@ffff907d91148b40 x1874670011695360/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828584.181290:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011695360/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828584.181301:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828584.181305:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828584.181310:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011695360:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828584.308707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828584.308817:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828585.332728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828585.332764:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828586.356728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828586.356834:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828587.380714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828587.380866:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828588.404776:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828588.404787:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828589.300826:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828589.300845:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828589.300849:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828589.300862:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3993 00000100:00080000:2.0:1787828589.300875:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828589.300882:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828589.300931:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011695488/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828589.300955:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828589.300960:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828589.300962:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828589.300967:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3993 00000100:00080000:2.0:1787828589.300972:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828589.300977:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828589.301002:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828589.301017:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011695616/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828589.301018:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011695488:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828589.301029:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828589.301032:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828589.301035:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828589.301039:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3993 00000100:00080000:2.0:1787828589.301043:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828589.301048:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828589.301054:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828589.301065:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011695744/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828589.301075:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828589.301079:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828589.301081:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828589.301084:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3993 00000100:00080000:2.0:1787828589.301097:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828589.301101:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:3.0:1787828589.301101:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:2.0:1787828589.301107:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828589.301118:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011695872/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828589.301149:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828589/real 1787828589] req@ffff907d91148b40 x1874670011695488/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828589.301287:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011695488/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828589.301318:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828589.301324:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828589.301330:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011695488:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828589.301510:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011695616:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828589.301581:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828589.301598:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828589/real 1787828589] req@ffff907d9114a940 x1874670011695616/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828589.301642:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011695616/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828589.301653:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828589.301656:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828589.301659:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011695616:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828589.301683:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011695744:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828589.301708:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828589.301722:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828589/real 1787828589] req@ffff907d91148f00 x1874670011695744/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828589.301744:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011695744/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828589.301750:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828589.301752:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828589.301755:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011695744:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828589.301759:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011695872:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828589.301774:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828589.301781:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828589/real 1787828589] req@ffff907d911492c0 x1874670011695872/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828589.301789:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011695872/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828589.301799:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828589.301801:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828589.301804:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011695872:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828589.428691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828589.428735:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828590.452683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828590.452775:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828591.476660:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828591.476735:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828592.500691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828592.500795:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828593.524714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828593.524790:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828594.420711:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828594.420718:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828594.420723:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828594.420732:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 3998 00000100:00080000:1.0:1787828594.420740:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828594.420746:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828594.420777:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011696000/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828594.420798:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828594.420800:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828594.420802:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828594.420804:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3998 00000100:00080000:1.0:1787828594.420806:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828594.420809:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828594.420816:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828594.420826:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011696128/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828594.420833:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828594.420836:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828594.420837:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828594.420839:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3998 00000100:00080000:1.0:1787828594.420841:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828594.420843:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828594.420846:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828594.420855:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011696256/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828594.420865:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828594.420868:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828594.420869:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828594.420872:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 3998 00000100:00080000:1.0:1787828594.420874:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828594.420876:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828594.420879:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828594.420888:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011696384/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828594.420895:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011696000:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787828594.420980:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828594.421037:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828594/real 1787828594] req@ffff907d908a21c0 x1874670011696000/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828594.421054:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011696000/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828594.421063:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828594.421067:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828594.421070:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011696000:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828594.421077:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011696128:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828594.421097:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828594.421106:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828594/real 1787828594] req@ffff907d908a2580 x1874670011696128/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828594.421114:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011696128/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828594.421121:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828594.421124:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828594.421126:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011696128:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828594.421130:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011696256:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828594.421142:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828594.421148:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828594/real 1787828594] req@ffff907d908a2940 x1874670011696256/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828594.421155:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011696256/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828594.421160:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828594.421162:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828594.421164:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011696256:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828594.421188:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011696384:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828594.421199:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828594.421206:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828594/real 1787828594] req@ffff907d908a0000 x1874670011696384/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828594.421214:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011696384/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828594.421219:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828594.421221:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828594.421223:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011696384:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828594.548764:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828594.548856:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828595.572769:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828595.572840:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828596.596738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828596.596779:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828597.620697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828597.620747:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828598.644679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828598.644729:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828599.540692:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828599.540702:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828599.540706:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828599.540715:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4003 00000100:00080000:1.0:1787828599.540725:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828599.540732:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828599.540769:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011696512/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828599.540794:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828599.540799:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828599.540802:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828599.540806:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4003 00000100:00080000:1.0:1787828599.540810:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828599.540814:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828599.540824:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828599.540839:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011696640/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828599.540848:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011696512:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828599.540854:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828599.540859:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828599.540861:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828599.540866:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4003 00000100:00080000:1.0:1787828599.540870:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828599.540875:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828599.540880:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828599.540895:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011696768/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828599.540905:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1787828599.540911:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787828599.540913:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828599.540916:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828599.540922:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4003 00000100:00080000:1.0:1787828599.540927:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828599.540931:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828599.540936:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828599.540950:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011696896/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828599.540957:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828599/real 1787828599] req@ffff907d908a1e00 x1874670011696512/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828599.540981:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011696512/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828599.540991:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828599.540995:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828599.540998:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011696512:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828599.541020:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011696640:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828599.541046:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828599.541061:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828599/real 1787828599] req@ffff907d908a2d00 x1874670011696640/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828599.541075:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011696640/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828599.541082:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828599.541084:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828599.541087:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011696640:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828599.541092:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011696768:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828599.541114:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828599.541127:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828599/real 1787828599] req@ffff907d908a0000 x1874670011696768/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828599.541140:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011696768/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828599.541149:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828599.541151:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828599.541154:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011696768:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828599.541158:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011696896:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828599.541173:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828599.541181:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828599/real 1787828599] req@ffff907d908a2940 x1874670011696896/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828599.541189:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011696896/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828599.541194:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828599.541196:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828599.541198:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011696896:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828599.668712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828599.668734:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828600.692709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828600.692761:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828601.716722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828601.716794:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828602.740714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828602.740791:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828603.764717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828603.764807:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828604.660695:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828604.660703:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828604.660712:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828604.660720:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4008 00000100:00080000:3.0:1787828604.660729:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828604.660735:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828604.660765:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011697024/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828604.660797:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828604.660801:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828604.660803:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828604.660809:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4008 00000100:00080000:3.0:1787828604.660813:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828604.660819:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828604.660831:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828604.660839:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011697024:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828604.660845:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011697152/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828604.660855:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828604.660858:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828604.660859:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828604.660862:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4008 00000100:00080000:3.0:1787828604.660866:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828604.660869:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828604.660874:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828604.660882:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011697280/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828604.660891:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787828604.660896:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787828604.660907:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828604.660909:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828604.660913:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4008 00000100:00080000:3.0:1787828604.660917:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828604.660921:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828604.660926:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828604.660930:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828604/real 1787828604] req@ffff907da562f480 x1874670011697024/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828604.660941:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011697408/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828604.660949:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011697024/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828604.660959:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828604.660962:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828604.660966:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011697024:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828604.660985:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011697152:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828604.661002:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828604.661011:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828604/real 1787828604] req@ffff907da562e580 x1874670011697152/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828604.661019:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011697152/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828604.661024:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828604.661027:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828604.661029:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011697152:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828604.661032:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011697280:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828604.661043:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828604.661050:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828604/real 1787828604] req@ffff907da562d2c0 x1874670011697280/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828604.661057:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011697280/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828604.661063:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828604.661064:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828604.661066:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011697280:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828604.661070:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011697408:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828604.661079:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828604.661085:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828604/real 1787828604] req@ffff907da562fc00 x1874670011697408/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828604.661092:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011697408/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828604.661097:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828604.661099:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828604.661101:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011697408:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828604.788677:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828604.788811:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828605.812703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828605.812768:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828606.836700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828606.836759:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828607.860748:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828607.860772:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828608.884716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828608.884803:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828609.780710:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828609.780718:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828609.780722:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828609.780731:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4013 00000100:00080000:1.0:1787828609.780743:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828609.780749:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828609.780784:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011697536/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828609.780806:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828609.780810:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828609.780812:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828609.780817:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4013 00000100:00080000:1.0:1787828609.780821:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828609.780824:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828609.780832:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828609.780842:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011697664/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828609.780850:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828609.780853:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828609.780854:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828609.780857:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4013 00000100:00080000:1.0:1787828609.780859:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828609.780862:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828609.780865:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828609.780872:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011697792/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828609.780881:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828609.780885:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828609.780887:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828609.780892:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4013 00000100:00080000:1.0:1787828609.780895:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828609.780898:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828609.780902:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828609.780910:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011697920/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828609.780920:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011697536:192.168.203.104@tcp:250:kworker.0 00000800:00000100:2.0:1787828609.781002:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828609.781050:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828609/real 1787828609] req@ffff907d908a2580 x1874670011697536/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828609.781071:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011697536/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828609.781082:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828609.781086:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828609.781089:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011697536:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828609.781097:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011697664:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828609.781113:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828609.781122:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828609/real 1787828609] req@ffff907d908a21c0 x1874670011697664/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828609.781130:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011697664/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828609.781136:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828609.781138:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828609.781140:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011697664:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828609.781144:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011697792:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828609.781156:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828609.781161:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828609/real 1787828609] req@ffff907d908a3840 x1874670011697792/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828609.781169:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011697792/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828609.781178:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828609.781181:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828609.781183:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011697792:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828609.781212:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011697920:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828609.781251:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828609.781265:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828609/real 1787828609] req@ffff907d908a1680 x1874670011697920/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828609.781278:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011697920/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828609.781285:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828609.781287:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828609.781289:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011697920:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828609.908711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828609.908768:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828610.932677:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828610.932756:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828611.956712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828611.956766:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828612.980741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828612.980828:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828614.004756:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828614.004856:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828614.900679:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828614.900690:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828614.900695:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828614.900707:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4018 00000100:00080000:1.0:1787828614.900719:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828614.900726:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828614.900769:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011698048/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828614.900801:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828614.900805:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828614.900808:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828614.900813:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4018 00000100:00080000:1.0:1787828614.900817:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828614.900822:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828614.900835:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828614.900849:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011698176/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828614.900857:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828614.900860:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828614.900862:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828614.900866:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4018 00000100:00080000:1.0:1787828614.900870:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828614.900872:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828614.900877:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828614.900884:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011698048:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828614.900891:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011698304/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828614.900909:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828614.900911:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828614.900912:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828614.900917:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4018 00000100:00080000:1.0:1787828614.900918:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828614.900922:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828614.900925:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828614.900935:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011698432/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828614.900975:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828614.901017:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828614/real 1787828614] req@ffff907d908a0f00 x1874670011698048/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828614.901036:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011698048/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828614.901049:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828614.901062:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828614.901066:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011698048:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828614.901075:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011698176:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828614.901097:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828614.901110:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828614/real 1787828614] req@ffff907d908a12c0 x1874670011698176/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828614.901123:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011698176/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828614.901132:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828614.901135:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828614.901138:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011698176:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828614.901165:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011698304:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828614.901186:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828614.901211:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828614/real 1787828614] req@ffff907d908a1680 x1874670011698304/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828614.901225:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011698304/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828614.901235:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828614.901239:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828614.901242:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011698304:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828614.901250:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011698432:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828614.901267:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828614.901278:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828614/real 1787828614] req@ffff907d908a3840 x1874670011698432/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828614.901290:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011698432/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828614.901298:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828614.901301:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828614.901305:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011698432:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828615.028725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828615.028807:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828616.052708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828616.052777:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828617.076704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828617.076753:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828618.100732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828618.100799:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828619.124725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828619.124814:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828620.020693:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828620.020703:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828620.020708:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828620.020719:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4023 00000100:00080000:1.0:1787828620.020731:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828620.020739:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828620.020777:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011698560/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828620.020809:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828620.020813:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828620.020814:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828620.020817:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4023 00000100:00080000:1.0:1787828620.020820:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828620.020823:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828620.020832:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828620.020843:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011698688/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828620.020852:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011698560:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828620.020854:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828620.020859:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828620.020862:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828620.020866:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4023 00000100:00080000:1.0:1787828620.020868:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828620.020870:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828620.020873:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828620.020881:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011698816/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828620.020887:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828620.020890:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828620.020891:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828620.020893:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4023 00000100:00080000:1.0:1787828620.020896:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828620.020898:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828620.020902:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828620.020908:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011698944/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828620.020922:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828620.020957:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828620/real 1787828620] req@ffff907d908a21c0 x1874670011698560/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828620.020970:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011698560/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828620.020978:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828620.020981:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828620.020983:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011698560:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828620.021007:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011698688:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828620.021024:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828620.021033:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828620/real 1787828620] req@ffff907d908a2580 x1874670011698688/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828620.021041:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011698688/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828620.021047:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828620.021049:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828620.021052:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011698688:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828620.021056:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011698816:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828620.021068:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828620.021074:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828620/real 1787828620] req@ffff907d908a2940 x1874670011698816/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828620.021081:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011698816/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828620.021086:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828620.021088:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828620.021090:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011698816:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828620.021093:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011698944:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828620.021103:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828620.021109:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828620/real 1787828620] req@ffff907d908a0000 x1874670011698944/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828620.021116:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011698944/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828620.021121:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828620.021123:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828620.021125:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011698944:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828620.148699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828620.148752:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828621.172714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828621.172786:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828622.196690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828622.196749:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828623.220698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828623.220768:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828624.244689:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828624.244752:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828625.140696:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828625.140711:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828625.140715:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828625.140723:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4028 00000100:00080000:2.0:1787828625.140733:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828625.140740:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828625.140771:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011699072/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828625.140793:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828625.140797:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828625.140798:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828625.140806:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4028 00000100:00080000:2.0:1787828625.140809:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828625.140813:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828625.140822:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828625.140827:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011699072:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828625.140837:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011699200/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828625.140848:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828625.140851:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828625.140852:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828625.140856:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4028 00000100:00080000:2.0:1787828625.140859:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828625.140862:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828625.140866:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828625.140875:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011699328/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828625.140882:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828625.140884:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828625.140886:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787828625.140891:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787828625.140894:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4028 00000100:00080000:2.0:1787828625.140898:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828625.140901:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828625.140905:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828625.140917:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011699456/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828625.140938:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828625/real 1787828625] req@ffff907d9114a580 x1874670011699072/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828625.140953:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011699072/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828625.140961:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828625.140964:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828625.140967:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011699072:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828625.140986:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011699200:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787828625.141007:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828625.141017:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828625/real 1787828625] req@ffff907d9114ad00 x1874670011699200/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828625.141027:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011699200/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828625.141033:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828625.141036:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828625.141038:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011699200:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828625.141043:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011699328:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828625.141055:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828625.141062:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828625/real 1787828625] req@ffff907d91149a40 x1874670011699328/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828625.141070:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011699328/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828625.141075:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828625.141077:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828625.141079:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011699328:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828625.141082:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011699456:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828625.141093:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828625.141099:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828625/real 1787828625] req@ffff907d9114b840 x1874670011699456/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828625.141106:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011699456/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828625.141111:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828625.141113:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828625.141115:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011699456:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828625.268695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828625.268760:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828626.292691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828626.292763:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828627.316692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828627.316776:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828628.340681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828628.340720:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828629.364724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828629.364748:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828630.260694:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828630.260703:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828630.260706:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828630.260715:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4034 00000100:00080000:1.0:1787828630.260724:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828630.260730:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828630.260761:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011699584/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828630.260784:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828630.260787:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828630.260788:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828630.260791:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4034 00000100:00080000:1.0:1787828630.260793:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828630.260797:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828630.260805:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828630.260814:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011699712/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828630.260820:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011699584:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828630.260825:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828630.260831:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828630.260833:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828630.260836:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4034 00000100:00080000:1.0:1787828630.260838:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828630.260842:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828630.260845:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828630.260854:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011699840/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828630.260860:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828630.260863:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828630.260864:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828630.260867:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4034 00000100:00080000:1.0:1787828630.260869:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828630.260871:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828630.260874:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828630.260881:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011699968/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828630.260897:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828630.260942:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828630/real 1787828630] req@ffff907d908a2d00 x1874670011699584/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828630.260961:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011699584/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828630.260972:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828630.260976:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828630.260981:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011699584:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828630.261003:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011699712:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828630.261027:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828630.261041:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828630/real 1787828630] req@ffff907d908a1e00 x1874670011699712/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828630.261053:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011699712/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828630.261062:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828630.261065:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828630.261069:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011699712:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828630.261076:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011699840:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828630.261117:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828630.261141:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828630/real 1787828630] req@ffff907d908a0000 x1874670011699840/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828630.261158:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011699840/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828630.261167:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828630.261171:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828630.261174:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011699840:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828630.261182:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011699968:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828630.261203:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828630.261216:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828630/real 1787828630] req@ffff907d908a2940 x1874670011699968/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828630.261228:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011699968/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828630.261237:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828630.261241:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828630.261244:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011699968:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828630.388735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828630.388793:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828631.412739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828631.412753:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828632.436685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828632.436759:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828633.460711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828633.460808:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828634.484731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828634.484773:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828635.380759:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828635.380797:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828635.380803:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828635.380817:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4039 00000100:00080000:1.0:1787828635.380833:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828635.380843:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828635.380942:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011700096/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828635.381009:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828635.381014:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828635.381017:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828635.381023:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4039 00000100:00080000:1.0:1787828635.381029:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828635.381045:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828635.381095:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828635.381115:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011700224/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828635.381124:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011700096:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828635.381135:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828635.381143:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828635.381163:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828635.381171:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4039 00000100:00080000:1.0:1787828635.381178:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828635.381186:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828635.381196:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828635.381224:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011700352/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828635.381258:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828635.381262:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828635.381264:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828635.381269:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4039 00000800:00000100:2.0:1787828635.381277:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787828635.381280:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828635.381289:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828635.381297:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828635.381326:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011700480/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828635.381356:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828635/real 1787828635] req@ffff907d908a2580 x1874670011700096/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828635.381388:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011700096/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828635.381400:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828635.381405:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828635.381409:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011700096:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828635.381441:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011700224:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828635.381466:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828635.381480:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828635/real 1787828635] req@ffff907d908a21c0 x1874670011700224/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828635.381502:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011700224/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828635.381511:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828635.381514:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828635.381518:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011700224:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828635.381523:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011700352:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828635.381540:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828635.381562:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828635/real 1787828635] req@ffff907d908a3840 x1874670011700352/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828635.381573:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011700352/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828635.381588:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828635.381591:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828635.381594:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011700352:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828635.381600:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011700480:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828635.381614:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828635.381660:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828635/real 1787828635] req@ffff907d908a1680 x1874670011700480/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828635.381674:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011700480/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828635.381682:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828635.381686:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828635.381689:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011700480:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828635.508744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828635.508834:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828636.532710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828636.532753:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828637.556682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828637.556757:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828638.580743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828638.580813:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828639.604735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828639.604796:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828640.500735:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828640.500743:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828640.500746:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828640.500754:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4044 00000100:00080000:1.0:1787828640.500764:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828640.500770:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828640.500803:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011700608/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828640.500826:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828640.500829:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828640.500832:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828640.500835:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4044 00000100:00080000:1.0:1787828640.500839:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828640.500841:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828640.500850:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828640.500861:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011700736/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828640.500875:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011700608:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828640.500876:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828640.500893:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828640.500897:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828640.500901:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4044 00000100:00080000:1.0:1787828640.500903:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828640.500906:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828640.500909:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828640.500921:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011700864/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828640.500928:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828640.500930:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828640.500931:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828640.500934:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4044 00000100:00080000:1.0:1787828640.500936:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828640.500938:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828640.500941:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828640.500948:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011700992/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828640.500987:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828640.501062:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828640/real 1787828640] req@ffff907d908a12c0 x1874670011700608/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828640.501085:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011700608/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828640.501098:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828640.501103:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828640.501108:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011700608:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828640.501140:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011700736:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828640.501169:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828640.501185:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828640/real 1787828640] req@ffff907d908a0f00 x1874670011700736/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828640.501198:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011700736/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828640.501206:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828640.501210:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828640.501213:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011700736:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828640.501220:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011700864:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828640.501241:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828640.501251:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828640/real 1787828640] req@ffff907d908a1680 x1874670011700864/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828640.501263:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011700864/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828640.501272:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828640.501275:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828640.501279:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011700864:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828640.501285:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011700992:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828640.501301:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828640.501312:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828640/real 1787828640] req@ffff907d908a3840 x1874670011700992/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828640.501323:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011700992/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828640.501331:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828640.501335:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828640.501338:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011700992:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828640.628760:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828640.628804:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828641.652714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828641.652780:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828642.676713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828642.676795:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828643.700684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828643.700765:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828644.724708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828644.724811:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828645.620692:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828645.620703:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828645.620708:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828645.620716:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4049 00000100:00080000:2.0:1787828645.620725:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828645.620731:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828645.620760:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011701120/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828645.620787:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828645.620790:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828645.620792:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828645.620795:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4049 00000100:00080000:2.0:1787828645.620797:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828645.620800:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828645.620809:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828645.620822:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011701248/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828645.620823:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011701120:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828645.620841:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828645.620844:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828645.620845:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828645.620847:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4049 00000100:00080000:2.0:1787828645.620850:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828645.620852:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828645.620855:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828645.620864:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011701376/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828645.620870:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828645.620872:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828645.620874:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828645.620876:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4049 00000100:00080000:2.0:1787828645.620879:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828645.620897:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828645.620901:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828645.620909:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011701504/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828645.620952:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011701248:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828645.620996:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011701376:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828645.621036:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011701504:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828645.748677:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828645.748744:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828646.772718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828646.772752:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828647.796708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828647.796820:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787828648.692867:0:2337:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:3.0:1787828648.692920:0:2337:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787828648.692942:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787828648.693003:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787828648.693016:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787828648.693028:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:1.0:1787828648.693062:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828645/real 1787828648] req@ffff907d9114b480 x1874670011701248/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828648.693116:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011701248/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828648.693132:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828648.693139:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828648.693145:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011701248:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828648.693156:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828645/real 1787828648] req@ffff907d91149680 x1874670011701376/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828648.693178:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011701376/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828648.693187:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828648.693191:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828648.693194:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011701376:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828648.693202:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828645/real 1787828648] req@ffff907d9114bc00 x1874670011701504/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828648.693217:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011701504/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828648.693226:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828648.693230:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828648.693235:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011701504:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828648.693242:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828645/real 1787828648] req@ffff907d911483c0 x1874670011701120/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828648.693258:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011701120/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828648.693267:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828648.693271:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828648.693275:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011701120:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828648.820703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828648.820754:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828649.844702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828649.844796:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828650.740784:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828650.740795:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828650.740799:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828650.740809:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4054 00000100:00080000:0.0:1787828650.740820:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828650.740827:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828650.740861:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011701632/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828650.740891:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828650.740895:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828650.740897:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828650.740901:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4054 00000100:00080000:0.0:1787828650.740904:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828650.740908:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828650.740918:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828650.740933:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011701760/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828650.740970:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828650.740977:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787828650.740979:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011701632:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828650.740981:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828650.740998:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4054 00000100:00080000:0.0:1787828650.741003:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828650.741008:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828650.741014:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828650.741031:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011701888/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828650.741042:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828650.741047:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828650.741049:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828650.741053:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4054 00000100:00080000:0.0:1787828650.741057:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828650.741063:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828650.741068:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828650.741080:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011702016/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828650.741129:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828650.741195:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828650/real 1787828650] req@ffff907d90846d00 x1874670011701632/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828650.741246:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011701632/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828650.741263:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828650.741270:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828650.741276:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011701632:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828650.741319:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011701760:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787828650.741356:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828650.741375:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828650/real 1787828650] req@ffff907d90845680 x1874670011701760/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828650.741392:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011701760/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828650.741405:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828650.741410:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828650.741416:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011701760:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828650.741427:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011701888:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828650.741459:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828650.741476:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828650/real 1787828650] req@ffff907d90844780 x1874670011701888/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828650.741492:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011701888/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828650.741503:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828650.741508:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828650.741512:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011701888:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828650.741522:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011702016:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828650.741569:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828650.741585:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828650/real 1787828650] req@ffff907d908443c0 x1874670011702016/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828650.741676:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011702016/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828650.741689:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828650.741696:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828650.741702:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011702016:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828650.868738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828650.868793:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828651.892658:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828651.892737:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828652.916732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828652.916758:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828653.940738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828653.940820:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828654.964724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828654.964815:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828655.860734:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828655.860745:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828655.860751:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828655.860763:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4059 00000100:00080000:2.0:1787828655.860776:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828655.860784:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828655.860830:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011702144/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828655.860855:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828655.860858:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828655.860859:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828655.860862:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4059 00000100:00080000:2.0:1787828655.860865:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828655.860867:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828655.860883:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828655.860894:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011702272/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828655.860903:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828655.860908:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011702144:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828655.860908:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828655.860922:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828655.860929:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4059 00000100:00080000:2.0:1787828655.860932:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828655.860935:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828655.860939:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828655.860951:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011702400/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828655.860964:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828655.860967:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828655.860968:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828655.860972:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4059 00000100:00080000:2.0:1787828655.860974:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828655.860977:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828655.860980:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828655.860990:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011702528/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828655.860998:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828655.861041:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828655/real 1787828655] req@ffff907d911483c0 x1874670011702144/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828655.861055:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011702144/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828655.861064:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828655.861068:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828655.861071:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011702144:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828655.861092:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011702272:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787828655.861109:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828655.861118:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828655/real 1787828655] req@ffff907d9114bc00 x1874670011702272/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828655.861126:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011702272/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828655.861132:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828655.861134:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828655.861137:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011702272:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787828655.861140:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011702400:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828655.861152:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828655.861158:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828655/real 1787828655] req@ffff907d91149680 x1874670011702400/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828655.861166:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011702400/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828655.861171:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828655.861173:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828655.861184:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011702400:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787828655.861190:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011702528:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787828655.861211:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828655.861223:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828655/real 1787828655] req@ffff907d9114b480 x1874670011702528/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828655.861236:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011702528/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828655.861246:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828655.861248:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787828655.861250:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011702528:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828655.988727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828655.988805:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828657.012697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828657.012725:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828658.036716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828658.036810:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828659.060726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828659.060812:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828660.084720:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828660.084766:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:2.0:1787828660.980735:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828660.980747:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828660.980752:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828660.980763:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4064 00000100:00080000:2.0:1787828660.980777:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828660.980786:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828660.980830:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011702656/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828660.980864:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828660.980869:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828660.980872:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828660.980889:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4064 00000100:00080000:2.0:1787828660.980894:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828660.980904:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828660.980919:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828660.980934:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011702656:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828660.980940:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011702784/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828660.980951:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828660.980954:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828660.980957:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828660.980960:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4064 00000100:00080000:2.0:1787828660.980964:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828660.980967:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828660.980971:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828660.980983:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011702912/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828660.980990:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828660.980993:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828660.980995:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828660.980998:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4064 00000100:00080000:2.0:1787828660.981010:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828660.981012:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828660.981016:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787828660.981027:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828660.981032:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011703040/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828660.981100:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828660/real 1787828660] req@ffff907d9114b840 x1874670011702656/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828660.981125:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011702656/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828660.981135:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828660.981139:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828660.981147:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011702656:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828660.981186:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011702784:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828660.981215:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828660.981227:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828660/real 1787828660] req@ffff907d91149a40 x1874670011702784/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828660.981242:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011702784/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828660.981248:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828660.981251:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828660.981255:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011702784:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828660.981260:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011702912:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828660.981277:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828660.981287:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828660/real 1787828660] req@ffff907d9114ad00 x1874670011702912/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828660.981297:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011702912/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828660.981304:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828660.981307:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828660.981309:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011702912:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828660.981315:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011703040:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828660.981337:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828660.981348:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828660/real 1787828660] req@ffff907d9114a580 x1874670011703040/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828660.981358:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011703040/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828660.981367:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828660.981369:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828660.981371:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011703040:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828661.108775:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828661.108894:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828662.132745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828662.132817:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828663.156762:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828663.156832:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828664.180729:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828664.180758:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787828665.204762:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828665.204828:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828666.100726:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828666.100737:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828666.100742:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828666.100753:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4069 00000100:00080000:3.0:1787828666.100765:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828666.100773:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828666.100816:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011703168/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828666.100862:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828666.100866:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828666.100868:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828666.100875:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4069 00000100:00080000:3.0:1787828666.100879:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828666.100885:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828666.100897:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828666.100913:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011703296/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828666.100914:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011703168:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828666.100930:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828666.100933:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828666.100935:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828666.100939:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4069 00000100:00080000:3.0:1787828666.100944:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828666.100949:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828666.100955:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828666.100970:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011703424/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828666.100988:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828666.100992:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1787828666.100997:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787828666.100998:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828666.101006:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4069 00000100:00080000:3.0:1787828666.101011:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828666.101017:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828666.101023:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828666.101037:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011703552/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828666.101056:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828666/real 1787828666] req@ffff907da562c000 x1874670011703168/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828666.101072:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011703168/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828666.101081:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828666.101084:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828666.101088:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011703168:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828666.101112:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011703296:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828666.101129:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828666.101138:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828666/real 1787828666] req@ffff907da562e940 x1874670011703296/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828666.101146:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011703296/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828666.101152:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828666.101155:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828666.101157:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011703296:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828666.101161:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011703424:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828666.101173:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828666.101179:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828666/real 1787828666] req@ffff907da562c3c0 x1874670011703424/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828666.101187:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011703424/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828666.101192:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828666.101194:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828666.101196:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011703424:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828666.101199:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011703552:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828666.101209:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828666.101215:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828666/real 1787828666] req@ffff907da562f0c0 x1874670011703552/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828666.101222:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011703552/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828666.101227:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828666.101228:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828666.101230:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011703552:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828666.228717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828666.228744:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828667.252714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828667.252744:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828668.276711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828668.276801:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828669.300737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828669.300807:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828670.324712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828670.324752:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828671.220690:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828671.220699:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828671.220704:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828671.220730:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4074 00000100:00080000:3.0:1787828671.220741:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828671.220747:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828671.220785:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011703680/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828671.220832:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828671.220836:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828671.220838:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828671.220842:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4074 00000100:00080000:3.0:1787828671.220846:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828671.220849:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828671.220876:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828671.220890:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011703808/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828671.220901:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828671.220906:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011703680:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828671.220906:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828671.220909:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828671.220920:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4074 00000100:00080000:3.0:1787828671.220924:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828671.220926:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828671.220930:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828671.220942:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011703936/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828671.220950:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828671.220953:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828671.220955:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828671.220958:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4074 00000100:00080000:3.0:1787828671.220961:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828671.220963:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828671.220983:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828671.220993:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011704064/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828671.221005:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828671.221043:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828671/real 1787828671] req@ffff907da562da40 x1874670011703680/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828671.221057:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011703680/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828671.221066:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828671.221069:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828671.221072:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011703680:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828671.221091:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011703808:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828671.221108:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828671.221117:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828671/real 1787828671] req@ffff907da562f840 x1874670011703808/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828671.221126:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011703808/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828671.221132:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828671.221134:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828671.221136:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011703808:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828671.221140:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011703936:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828671.221152:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828671.221158:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828671/real 1787828671] req@ffff907da562e1c0 x1874670011703936/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828671.221171:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011703936/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828671.221177:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828671.221179:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828671.221181:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011703936:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828671.221184:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011704064:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828671.221195:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828671.221208:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828671/real 1787828671] req@ffff907da562cb40 x1874670011704064/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828671.221216:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011704064/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828671.221222:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828671.221224:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828671.221226:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011704064:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828671.348672:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828671.348701:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828672.372715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828672.372783:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828673.396698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828673.396745:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828674.420712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828674.420744:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828675.444813:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828675.444892:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828676.340706:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828676.340717:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828676.340721:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828676.340732:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4080 00000100:00080000:2.0:1787828676.340744:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828676.340752:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828676.340847:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011704192/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828676.340877:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828676.340879:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828676.340881:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828676.340884:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4080 00000100:00080000:2.0:1787828676.340887:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828676.340892:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828676.340902:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828676.340913:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011704320/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828676.340922:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828676.340926:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011704192:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828676.340927:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828676.340931:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828676.340938:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4080 00000100:00080000:2.0:1787828676.340940:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828676.340943:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828676.340947:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828676.340957:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011704448/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828676.340964:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828676.340967:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828676.340968:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828676.340971:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4080 00000100:00080000:2.0:1787828676.340973:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828676.340976:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828676.340979:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828676.340988:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011704576/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828676.341011:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828676.341051:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828676/real 1787828676] req@ffff907d911492c0 x1874670011704192/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828676.341072:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011704192/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828676.341082:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828676.341085:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828676.341089:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011704192:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828676.341112:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011704320:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828676.341130:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828676.341138:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828676/real 1787828676] req@ffff907d91148f00 x1874670011704320/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828676.341147:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011704320/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828676.341153:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828676.341155:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828676.341157:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011704320:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828676.341162:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011704448:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828676.341174:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828676.341180:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828676/real 1787828676] req@ffff907d9114a940 x1874670011704448/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828676.341187:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011704448/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828676.341193:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828676.341200:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828676.341202:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011704448:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828676.341207:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011704576:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828676.341219:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828676.341225:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828676/real 1787828676] req@ffff907d91148b40 x1874670011704576/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828676.341233:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011704576/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828676.341238:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828676.341240:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828676.341242:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011704576:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828676.468716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828676.468765:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828677.492860:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828677.492923:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787828678.516710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828678.516814:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828679.540725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828679.540771:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828680.564713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828680.564758:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828681.460674:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828681.460682:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828681.460685:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828681.460694:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4085 00000100:00080000:2.0:1787828681.460703:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828681.460708:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828681.460757:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011704704/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828681.460777:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828681.460780:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828681.460782:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828681.460785:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4085 00000100:00080000:2.0:1787828681.460788:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828681.460790:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828681.460798:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828681.460809:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011704832/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828681.460817:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011704704:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828681.460820:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828681.460825:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828681.460826:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828681.460830:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4085 00000100:00080000:2.0:1787828681.460833:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828681.460836:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828681.460839:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828681.460847:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011704960/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828681.460854:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828681.460857:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828681.460858:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828681.460861:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4085 00000100:00080000:2.0:1787828681.460863:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828681.460865:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828681.460879:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828681.460887:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011705088/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828681.460896:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828681.460957:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828681/real 1787828681] req@ffff907d91148b40 x1874670011704704/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828681.460971:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011704704/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828681.460979:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828681.460982:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828681.460986:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011704704:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828681.461014:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011704832:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828681.461032:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828681.461042:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828681/real 1787828681] req@ffff907d9114a940 x1874670011704832/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828681.461050:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011704832/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828681.461056:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828681.461058:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828681.461061:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011704832:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828681.461064:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011704960:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828681.461076:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828681.461083:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828681/real 1787828681] req@ffff907d91148f00 x1874670011704960/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828681.461090:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011704960/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828681.461095:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828681.461097:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828681.461099:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011704960:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828681.461102:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011705088:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828681.461112:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828681.461117:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828681/real 1787828681] req@ffff907d911492c0 x1874670011705088/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828681.461124:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011705088/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828681.461140:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828681.461142:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828681.461145:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011705088:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828681.588699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828681.588748:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828682.612799:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828682.612855:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828683.636804:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828683.636853:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828684.660683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828684.660771:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828685.684804:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828685.684806:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828686.580784:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828686.580798:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828686.580804:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828686.580816:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4090 00000100:00080000:1.0:1787828686.580830:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828686.580839:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828686.580887:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011705216/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828686.580952:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828686.580957:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828686.580959:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828686.580964:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4090 00000100:00080000:1.0:1787828686.580969:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828686.580981:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828686.580993:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828686.581012:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011705216:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828686.581014:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011705344/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828686.581031:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828686.581035:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828686.581037:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828686.581043:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4090 00000100:00080000:1.0:1787828686.581073:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828686.581083:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828686.581092:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787828686.581117:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828686.581119:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011705472/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828686.581152:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828686.581157:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828686.581160:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828686.581165:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4090 00000100:00080000:1.0:1787828686.581171:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828686.581176:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828686.581183:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828686.581190:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828686/real 1787828686] req@ffff907d908a21c0 x1874670011705216/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828686.581201:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011705600/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828686.581253:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011705216/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828686.581268:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828686.581274:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828686.581279:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011705216:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828686.581305:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011705344:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828686.581328:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828686.581340:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828686/real 1787828686] req@ffff907d908a2580 x1874670011705344/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828686.581349:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011705344/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828686.581384:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828686.581388:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828686.581392:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011705344:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828686.581398:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011705472:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828686.581418:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828686.581428:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828686/real 1787828686] req@ffff907d908a2940 x1874670011705472/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828686.581439:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011705472/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828686.581447:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828686.581450:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828686.581453:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011705472:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828686.581458:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011705600:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828686.581473:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828686.581482:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828686/real 1787828686] req@ffff907d908a0000 x1874670011705600/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828686.581493:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011705600/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828686.581501:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828686.581503:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828686.581506:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011705600:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828686.708738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828686.708797:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828687.732754:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828687.732835:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828688.756763:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828688.756796:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828689.780694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828689.780749:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828690.804694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828690.804778:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828691.700751:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828691.700762:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828691.700768:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828691.700779:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4095 00000100:00080000:3.0:1787828691.700792:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828691.700799:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828691.700841:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011705728/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828691.700869:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828691.700872:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828691.700874:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828691.700877:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4095 00000100:00080000:3.0:1787828691.700879:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828691.700882:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828691.700892:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828691.700904:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011705856/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828691.700913:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828691.700916:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828691.700917:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828691.700920:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4095 00000100:00080000:3.0:1787828691.700922:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828691.700925:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828691.700929:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828691.700936:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011705984/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828691.700943:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828691.700946:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828691.700947:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828691.700950:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4095 00000100:00080000:3.0:1787828691.700951:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828691.700956:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828691.700961:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828691.700964:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011705728:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828691.700972:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011706112/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828691.701049:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828691.701090:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828691/real 1787828691] req@ffff907da562de00 x1874670011705728/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828691.701104:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011705728/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828691.701113:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828691.701118:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828691.701121:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011705728:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828691.701126:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011705856:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828691.701143:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828691.701151:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828691/real 1787828691] req@ffff907da562cb40 x1874670011705856/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828691.701160:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011705856/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828691.701166:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828691.701169:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828691.701171:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011705856:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828691.701192:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011705984:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828691.701205:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828691.701212:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828691/real 1787828691] req@ffff907da562e1c0 x1874670011705984/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828691.701220:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011705984/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828691.701231:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828691.701233:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828691.701236:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011705984:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828691.701239:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011706112:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828691.701249:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828691.701255:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828691/real 1787828691] req@ffff907da562f840 x1874670011706112/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828691.701263:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011706112/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828691.701268:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828691.701270:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828691.701272:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011706112:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828691.828695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828691.828771:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828692.852746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828692.852765:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828693.876720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828693.876750:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828694.900714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828694.900757:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828695.924701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828695.924754:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828696.820764:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828696.820773:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828696.820777:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828696.820787:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4100 00000100:00080000:1.0:1787828696.820797:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828696.820820:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828696.820860:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011706240/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828696.820891:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828696.820895:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828696.820897:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828696.820901:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4100 00000100:00080000:1.0:1787828696.820906:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828696.820910:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828696.820920:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828696.820938:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011706240:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828696.820947:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011706368/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828696.820959:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828696.820962:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828696.820964:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828696.820968:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4100 00000100:00080000:1.0:1787828696.820972:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828696.820975:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828696.820979:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828696.820990:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011706496/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828696.821008:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1787828696.821013:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787828696.821014:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828696.821018:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828696.821023:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4100 00000100:00080000:1.0:1787828696.821028:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828696.821031:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828696.821035:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828696.821043:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011706624/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828696.821056:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828696/real 1787828696] req@ffff907d908a1e00 x1874670011706240/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828696.821073:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011706240/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828696.821083:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828696.821086:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828696.821091:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011706240:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828696.821362:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011706368:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828696.821403:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828696.821508:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828696/real 1787828696] req@ffff907d908a2d00 x1874670011706368/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828696.821540:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011706368/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828696.821547:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828696.821550:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828696.821553:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011706368:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828696.821559:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011706496:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828696.821577:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828696.821585:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828696/real 1787828696] req@ffff907d908a0000 x1874670011706496/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828696.821593:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011706496/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828696.821598:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828696.821601:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828696.821603:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011706496:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828696.821606:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011706624:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828696.821616:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828696.821660:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828696/real 1787828696] req@ffff907d908a2940 x1874670011706624/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828696.821701:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011706624/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828696.821710:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828696.821714:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828696.821717:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011706624:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828696.948748:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828696.948802:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828697.972743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828697.972797:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828698.996740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828698.996829:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828700.020682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828700.020752:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828701.044823:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828701.044870:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828701.940731:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828701.940745:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828701.940750:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828701.940760:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4105 00000100:00080000:3.0:1787828701.940773:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828701.940780:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828701.940821:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011706752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828701.940857:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828701.940860:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828701.940861:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828701.940865:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4105 00000100:00080000:3.0:1787828701.940867:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828701.940869:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828701.940879:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828701.940889:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011706880/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828701.940898:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828701.940901:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828701.940902:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828701.940905:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4105 00000100:00080000:3.0:1787828701.940907:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828701.940911:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828701.940913:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011706752:192.168.203.104@tcp:250:kworker.0 00000008:00100000:3.0:1787828701.940915:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828701.940927:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011707008/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828701.940934:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828701.940936:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828701.940938:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828701.940941:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4105 00000100:00080000:3.0:1787828701.940943:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828701.940946:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828701.940949:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828701.940959:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011707136/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828701.940991:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828701.941042:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828701/real 1787828701] req@ffff907da562da40 x1874670011706752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828701.941058:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011706752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828701.941067:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828701.941071:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828701.941074:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011706752:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828701.941081:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011706880:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828701.941101:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828701.941114:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828701/real 1787828701] req@ffff907da562f0c0 x1874670011706880/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828701.941128:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011706880/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828701.941137:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828701.941141:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828701.941145:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011706880:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828701.941174:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011707008:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828701.941192:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828701.941200:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828701/real 1787828701] req@ffff907da562c3c0 x1874670011707008/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828701.941208:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011707008/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828701.941214:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828701.941217:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828701.941219:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011707008:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828701.941223:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011707136:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828701.941234:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828701.941240:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828701/real 1787828701] req@ffff907da562e940 x1874670011707136/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828701.941247:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011707136/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828701.941253:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828701.941254:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828701.941256:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011707136:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828702.068726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828702.068768:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828703.092749:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828703.092856:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828704.116694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828704.116809:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828705.140709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828705.140758:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828706.164751:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828706.164846:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828707.060771:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828707.060787:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828707.060795:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828707.060811:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4110 00000100:00080000:3.0:1787828707.060827:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828707.060839:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828707.060897:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011707264/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828707.060945:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828707.060953:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828707.060958:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828707.060969:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4110 00000100:00080000:3.0:1787828707.060977:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828707.060987:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828707.060999:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011707264:192.168.203.104@tcp:250:kworker.0 00000008:00100000:3.0:1787828707.061005:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828707.061044:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011707392/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828707.061066:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828707.061072:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828707.061075:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787828707.061085:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787828707.061091:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4110 00000100:00080000:3.0:1787828707.061102:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828707.061109:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828707.061118:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828707.061141:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011707520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828707.061143:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828707/real 1787828707] req@ffff907da562c000 x1874670011707264/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828707.061227:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828707.061229:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011707264/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828707.061237:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828707.061244:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828707.061263:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4110 00000100:00080000:0.0:1787828707.061267:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828707.061272:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828707.061277:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828707.061287:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011707264:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828707.061295:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828707.061308:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828707.061317:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011707392:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828707.061336:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011707648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828707.061345:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828707.061361:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828707/real 1787828707] req@ffff907da562fc00 x1874670011707392/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828707.061391:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011707392/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828707.061398:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828707.061401:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828707.061403:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011707392:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828707.061410:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011707520:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828707.061427:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828707.061434:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828707/real 1787828707] req@ffff907da562d2c0 x1874670011707520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828707.061442:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011707520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828707.061448:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828707.061450:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828707.061453:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011707520:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828707.061464:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011707648:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828707.061474:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828707.061480:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828707/real 1787828707] req@ffff907da562e580 x1874670011707648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828707.061496:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011707648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828707.061509:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828707.061511:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828707.061513:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011707648:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828707.188772:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828707.188811:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828708.212705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828708.212782:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828709.236708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828709.236798:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828710.260680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828710.260793:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828711.284698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828711.284758:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828712.180711:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828712.180725:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828712.180731:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828712.180743:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4115 00000100:00080000:0.0:1787828712.180757:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828712.180767:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828712.180817:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011707776/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828712.180849:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828712.180853:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828712.180856:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828712.180862:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4115 00000100:00080000:0.0:1787828712.180867:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828712.180873:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828712.180884:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828712.180885:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011707776:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828712.180916:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011707904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828712.180929:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828712.180933:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828712.180935:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828712.180939:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4115 00000100:00080000:0.0:1787828712.180942:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828712.180946:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828712.180950:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828712.180962:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011708032/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828712.180971:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828712.180975:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828712.180977:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828712.180979:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4115 00000100:00080000:0.0:1787828712.180982:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828712.180985:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828712.180988:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828712.180997:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011708160/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828712.181067:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011707904:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828712.181099:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011708032:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828712.181114:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011708160:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828712.308711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828712.308757:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828713.332687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828713.332748:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828714.356696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828714.356739:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787828715.252910:0:2338:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:3.0:1787828715.252945:0:2338:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787828715.252961:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787828715.253000:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787828715.253011:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787828715.253020:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:0.0:1787828715.253051:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828712/real 1787828715] req@ffff907d908452c0 x1874670011707904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828715.253084:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011707904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828715.253096:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828715.253102:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828715.253108:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011707904:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828715.253116:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828712/real 1787828715] req@ffff907d90844000 x1874670011708032/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828715.253130:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011708032/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828715.253140:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828715.253144:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828715.253148:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011708032:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828715.253171:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828712/real 1787828715] req@ffff907d90845e00 x1874670011708160/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828715.253185:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011708160/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828715.253194:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828715.253198:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828715.253201:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011708160:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828715.253208:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828712/real 1787828715] req@ffff907d90844b40 x1874670011707776/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828715.253238:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011707776/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828715.253247:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828715.253250:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828715.253255:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011707776:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828715.380697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828715.380721:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828716.404705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828716.404772:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828717.300734:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828717.300750:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828717.300756:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828717.300770:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4121 00000100:00080000:3.0:1787828717.300785:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828717.300795:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828717.300845:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011708288/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828717.300883:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828717.300888:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828717.300890:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828717.300896:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4121 00000100:00080000:3.0:1787828717.300902:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828717.300908:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828717.300922:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828717.300935:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011708288:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828717.300963:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011708416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828717.300979:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828717.300984:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828717.300986:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828717.300991:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4121 00000100:00080000:3.0:1787828717.300996:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828717.301002:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828717.301008:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828717.301023:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011708544/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828717.301033:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787828717.301044:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828717.301050:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828717.301052:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828717.301059:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4121 00000100:00080000:3.0:1787828717.301067:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828717.301072:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828717.301079:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828717.301093:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828717/real 1787828717] req@ffff907da562f480 x1874670011708288/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828717.301096:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011708672/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828717.301151:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011708288/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828717.301165:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828717.301170:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828717.301174:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011708288:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828717.301202:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011708416:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828717.301231:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828717.301245:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828717/real 1787828717] req@ffff907da562c780 x1874670011708416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828717.301259:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011708416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828717.301267:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828717.301270:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828717.301274:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011708416:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828717.301282:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011708544:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828717.301305:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828717.301320:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828717/real 1787828717] req@ffff907da562ed00 x1874670011708544/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828717.301333:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011708544/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828717.301342:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828717.301346:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828717.301350:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011708544:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828717.301358:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011708672:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828717.301381:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828717.301394:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828717/real 1787828717] req@ffff907da562cf00 x1874670011708672/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828717.301406:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011708672/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828717.301415:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828717.301419:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828717.301422:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011708672:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828717.428734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828717.428801:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828718.452720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828718.452774:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828719.476750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828719.476838:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828720.500742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828720.500809:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828721.524720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828721.524758:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828722.420798:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828722.420808:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828722.420813:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828722.420825:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4126 00000100:00080000:0.0:1787828722.420837:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828722.420844:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828722.420888:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011708800/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828722.420918:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828722.420921:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828722.420923:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828722.420926:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4126 00000100:00080000:0.0:1787828722.420929:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828722.420932:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828722.420942:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828722.420952:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011708928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828722.420961:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828722.420963:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011708800:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828722.420964:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828722.420968:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828722.420976:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4126 00000100:00080000:0.0:1787828722.420979:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828722.420982:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828722.420984:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828722.420992:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011709056/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828722.420999:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828722.421001:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828722.421002:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828722.421006:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4126 00000100:00080000:0.0:1787828722.421009:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828722.421011:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828722.421015:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828722.421022:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011709184/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828722.421054:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828722.421125:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828722/real 1787828722] req@ffff907d90844b40 x1874670011708800/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828722.421148:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011708800/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828722.421162:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828722.421168:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828722.421173:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011708800:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828722.421204:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011708928:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828722.421247:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828722.421259:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828722/real 1787828722] req@ffff907d90845e00 x1874670011708928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828722.421268:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011708928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828722.421274:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828722.421276:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828722.421279:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011708928:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828722.421283:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011709056:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828722.421320:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828722.421328:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828722/real 1787828722] req@ffff907d90844000 x1874670011709056/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828722.421337:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011709056/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828722.421343:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828722.421345:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828722.421347:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011709056:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828722.421351:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011709184:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828722.421364:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828722.421370:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828722/real 1787828722] req@ffff907d908452c0 x1874670011709184/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828722.421379:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011709184/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828722.421385:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828722.421387:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828722.421389:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011709184:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828722.548769:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828722.548830:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828723.572749:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828723.572827:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828724.596714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828724.596808:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828725.620742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828725.620823:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828726.644701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828726.644787:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828727.540733:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828727.540747:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828727.540753:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828727.540766:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4131 00000100:00080000:0.0:1787828727.540781:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828727.540789:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828727.540840:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011709312/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828727.540874:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828727.540877:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828727.540879:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828727.540882:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4131 00000100:00080000:0.0:1787828727.540885:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828727.540889:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828727.540900:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828727.540913:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011709440/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828727.540921:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828727.540925:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828727.540927:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828727.540931:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4131 00000100:00080000:0.0:1787828727.540933:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828727.540937:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828727.540946:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828727.540946:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011709312:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828727.540964:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011709568/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828727.540973:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828727.540977:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828727.540978:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828727.540981:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4131 00000100:00080000:0.0:1787828727.540983:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828727.540985:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828727.540989:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828727.541002:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011709696/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828727.541022:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828727.541060:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828727/real 1787828727] req@ffff907d908470c0 x1874670011709312/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828727.541074:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011709312/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828727.541083:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828727.541087:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828727.541089:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011709312:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828727.541095:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011709440:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828727.541110:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828727.541118:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828727/real 1787828727] req@ffff907d90846580 x1874670011709440/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828727.541126:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011709440/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828727.541131:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828727.541133:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828727.541135:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011709440:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828727.541154:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011709568:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828727.541165:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828727.541172:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828727/real 1787828727] req@ffff907d90847c00 x1874670011709568/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828727.541179:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011709568/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828727.541185:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828727.541187:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828727.541189:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011709568:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828727.541192:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011709696:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828727.541210:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828727.541217:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828727/real 1787828727] req@ffff907d908452c0 x1874670011709696/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828727.541225:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011709696/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828727.541230:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828727.541232:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828727.541234:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011709696:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828727.668691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828727.668805:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828728.692770:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828728.692775:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787828729.716772:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828729.716830:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828730.740695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828730.740743:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828731.764712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828731.764733:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828732.660741:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828732.660756:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828732.660763:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828732.660776:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4136 00000100:00080000:0.0:1787828732.660793:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828732.660803:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828732.660851:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011709824/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828732.660893:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828732.660898:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828732.660901:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828732.660907:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4136 00000100:00080000:0.0:1787828732.660913:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828732.660918:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828732.660932:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828732.660945:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011709824:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828732.660952:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011709952/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828732.660970:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828732.660974:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828732.660977:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828732.660983:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4136 00000100:00080000:0.0:1787828732.660988:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828732.660993:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828732.661000:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828732.661014:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011710080/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828732.661026:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1787828732.661037:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787828732.661038:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828732.661043:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828732.661049:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4136 00000100:00080000:0.0:1787828732.661055:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828732.661061:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828732.661066:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828732.661095:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011710208/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828732.661096:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828732/real 1787828732] req@ffff907d90844000 x1874670011709824/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828732.661174:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011709824/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828732.661205:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828732.661210:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828732.661217:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011709824:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828732.661242:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011709952:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828732.661294:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828732.661330:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828732/real 1787828732] req@ffff907d90845e00 x1874670011709952/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828732.661343:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011709952/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828732.661355:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828732.661357:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828732.661360:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011709952:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828732.661366:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011710080:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828732.661413:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828732.661425:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828732/real 1787828732] req@ffff907d90844b40 x1874670011710080/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828732.661435:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011710080/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828732.661443:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828732.661448:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828732.661453:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011710080:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828732.661459:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011710208:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828732.661485:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828732.661512:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828732/real 1787828732] req@ffff907d908443c0 x1874670011710208/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828732.661526:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011710208/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828732.661538:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828732.661543:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828732.661627:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011710208:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828732.788685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828732.788855:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828733.812774:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828733.812853:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787828734.836777:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828734.836836:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787828735.860763:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828735.860952:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828736.884742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828736.884815:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828737.780733:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828737.780744:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828737.780750:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828737.780763:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4141 00000100:00080000:0.0:1787828737.780776:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828737.780783:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828737.780828:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011710336/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828737.780853:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828737.780856:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828737.780858:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828737.780861:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4141 00000100:00080000:0.0:1787828737.780864:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828737.780866:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828737.780875:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828737.780885:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011710464/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828737.780893:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828737.780898:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828737.780899:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828737.780902:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4141 00000100:00080000:0.0:1787828737.780904:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828737.780906:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828737.780910:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828737.780917:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011710592/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828737.780923:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828737.780926:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828737.780927:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828737.780932:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4141 00000100:00080000:0.0:1787828737.780936:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828737.780940:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828737.780946:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828737.780948:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011710336:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828737.780961:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011710720/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828737.781025:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828737.781069:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828737/real 1787828737] req@ffff907d90844780 x1874670011710336/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828737.781084:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011710336/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828737.781093:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828737.781096:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828737.781100:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011710336:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828737.781106:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011710464:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828737.781121:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828737.781129:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828737/real 1787828737] req@ffff907d90845680 x1874670011710464/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828737.781137:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011710464/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828737.781143:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828737.781145:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828737.781148:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011710464:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828737.781152:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011710592:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828737.781164:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828737.781170:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828737/real 1787828737] req@ffff907d90846d00 x1874670011710592/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828737.781177:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011710592/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828737.781183:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828737.781184:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828737.781186:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011710592:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828737.781208:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011710720:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828737.781218:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828737.781244:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828737/real 1787828737] req@ffff907d90846940 x1874670011710720/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828737.781252:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011710720/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828737.781258:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828737.781260:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828737.781262:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011710720:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828737.908723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828737.908786:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828738.932718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828738.932824:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828739.956785:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828739.956810:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828740.980679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828740.980776:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828742.004722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828742.004780:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828742.900754:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828742.900765:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828742.900771:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828742.900782:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4146 00000100:00080000:1.0:1787828742.900794:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828742.900802:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828742.900843:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011710848/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828742.900873:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828742.900877:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828742.900879:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828742.900883:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4146 00000100:00080000:1.0:1787828742.900886:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828742.900889:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828742.900899:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828742.900912:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011710976/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828742.900924:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828742.900929:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828742.900933:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787828742.900933:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011710848:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828742.900940:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4146 00000100:00080000:1.0:1787828742.900947:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828742.900953:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828742.900958:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828742.900969:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011711104/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828742.900978:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828742.900982:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828742.900984:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828742.900987:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4146 00000100:00080000:1.0:1787828742.900991:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828742.900994:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828742.900997:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828742.901007:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011711232/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828742.901032:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828742.901080:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828742/real 1787828742] req@ffff907d908a2580 x1874670011710848/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828742.901099:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011710848/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828742.901124:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828742.901128:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828742.901132:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011710848:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828742.901157:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011710976:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828742.901196:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828742.901210:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828742/real 1787828742] req@ffff907d908a21c0 x1874670011710976/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828742.901222:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011710976/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828742.901231:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828742.901234:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828742.901238:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011710976:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828742.901244:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011711104:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828742.901262:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828742.901270:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828742/real 1787828742] req@ffff907d908a3840 x1874670011711104/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828742.901282:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011711104/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828742.901290:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828742.901293:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828742.901296:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011711104:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828742.901301:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011711232:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828742.901342:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828742.901354:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828742/real 1787828742] req@ffff907d908a1680 x1874670011711232/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828742.901365:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011711232/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828742.901374:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828742.901377:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828742.901380:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011711232:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828743.028727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828743.028793:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828744.052776:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828744.052887:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828745.076692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828745.076757:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828746.100769:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828746.100872:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828747.124790:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828747.124871:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828748.020718:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828748.020726:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828748.020730:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828748.020739:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4151 00000100:00080000:1.0:1787828748.020749:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828748.020755:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828748.020786:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011711360/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828748.020808:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828748.020811:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828748.020813:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828748.020815:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4151 00000100:00080000:1.0:1787828748.020818:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828748.020820:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828748.020827:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828748.020835:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011711488/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828748.020860:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828748.020863:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828748.020865:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828748.020869:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4151 00000100:00080000:1.0:1787828748.020872:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828748.020878:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828748.020882:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828748.020892:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011711616/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828748.020907:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828748.020911:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011711360:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828748.020912:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828748.020916:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828748.020938:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4151 00000100:00080000:1.0:1787828748.020940:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828748.020943:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828748.020947:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828748.020958:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011711744/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828748.021008:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828748.021056:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828748/real 1787828748] req@ffff907d908a0f00 x1874670011711360/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828748.021087:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011711360/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828748.021096:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828748.021100:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828748.021103:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011711360:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828748.021109:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011711488:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828748.021126:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828748.021135:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828748/real 1787828748] req@ffff907d908a12c0 x1874670011711488/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828748.021143:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011711488/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828748.021174:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828748.021176:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828748.021179:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011711488:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828748.021240:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011711616:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828748.021267:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828748.021328:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828748/real 1787828748] req@ffff907d908a1680 x1874670011711616/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828748.021337:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011711616/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828748.021348:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828748.021352:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828748.021355:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011711616:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828748.021360:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011711744:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828748.021374:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828748.021380:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828748/real 1787828748] req@ffff907d908a3840 x1874670011711744/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828748.021388:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011711744/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828748.021393:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828748.021395:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828748.021397:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011711744:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828748.148703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828748.148844:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828749.172679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828749.172946:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828750.196791:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828750.196916:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828751.220738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828751.220795:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828752.244769:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828752.244982:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828753.140726:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828753.140736:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828753.140741:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828753.140751:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4156 00000100:00080000:3.0:1787828753.140761:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828753.140768:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828753.140804:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011711872/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828753.140841:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828753.140846:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828753.140848:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828753.140854:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4156 00000100:00080000:3.0:1787828753.140859:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828753.140865:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828753.140878:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828753.140899:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011712000/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828753.140909:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828753.140912:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828753.140914:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828753.140917:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4156 00000100:00080000:3.0:1787828753.140921:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828753.140925:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828753.140930:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828753.140944:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011712128/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828753.140970:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828753.140972:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011711872:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828753.140977:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828753.140985:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828753.140992:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4156 00000100:00080000:3.0:1787828753.140996:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828753.140998:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828753.141002:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828753.141014:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011712256/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828753.141152:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828753.141269:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828753/real 1787828753] req@ffff907da562d680 x1874670011711872/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828753.141314:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011711872/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828753.141328:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828753.141333:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828753.141338:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011711872:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828753.141347:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011712000:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828753.141402:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828753.141436:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828753/real 1787828753] req@ffff907da562cf00 x1874670011712000/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828753.141450:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011712000/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828753.141460:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828753.141464:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828753.141467:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011712000:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828753.141525:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011712128:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828753.141576:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828753.141599:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828753/real 1787828753] req@ffff907da562ed00 x1874670011712128/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828753.141618:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011712128/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828753.141629:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828753.141633:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828753.141637:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011712128:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828753.141646:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011712256:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828753.141800:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828753.141819:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828753/real 1787828753] req@ffff907da562c780 x1874670011712256/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828753.141832:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011712256/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828753.141840:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828753.141844:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828753.141847:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011712256:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828753.268751:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828753.268850:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828754.292683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828754.292831:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828755.316704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828755.316792:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828756.340685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828756.340732:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828757.364675:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828757.364788:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828758.260810:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828758.260821:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828758.260827:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828758.260855:0:61332:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4162 00000100:00080000:3.0:1787828758.260866:0:61332:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828758.260885:0:61332:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828758.260922:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011712384/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828758.260965:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828758.260969:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828758.260972:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828758.260976:0:61332:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4162 00000100:00080000:3.0:1787828758.260980:0:61332:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828758.260985:0:61332:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828758.260995:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828758.261013:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011712384:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828758.261015:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011712512/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828758.261029:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828758.261033:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828758.261035:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828758.261049:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4162 00000100:00080000:3.0:1787828758.261069:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828758.261075:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828758.261085:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828758.261105:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011712640/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828758.261113:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787828758.261145:0:61332:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828758.261151:0:61332:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828758.261154:0:61332:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828758.261161:0:61332:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4162 00000100:00080000:3.0:1787828758.261167:0:61332:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828758.261174:0:61332:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828758.261180:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828758/real 1787828758] req@ffff907da562f480 x1874670011712384/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787828758.261184:0:61332:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828758.261230:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011712384/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828758.261238:0:61332:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011712768/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828758.261265:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828758.261273:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828758.261280:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011712384:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828758.261306:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011712512:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828758.261366:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828758.261383:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828758/real 1787828758] req@ffff907da562e580 x1874670011712512/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828758.261411:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011712512/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828758.261418:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828758.261421:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828758.261423:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011712512:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828758.261428:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011712640:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828758.261445:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828758.261453:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828758/real 1787828758] req@ffff907da562d2c0 x1874670011712640/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828758.261462:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011712640/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828758.261484:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828758.261486:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828758.261488:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011712640:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828758.261492:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011712768:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828758.261510:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828758.261521:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828758/real 1787828758] req@ffff907da562fc00 x1874670011712768/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828758.261553:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011712768/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828758.261567:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828758.261571:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828758.261575:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011712768:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828758.388723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828758.388767:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828759.412708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828759.412798:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828760.436726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828760.436766:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828761.460746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828761.460787:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828762.484759:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828762.484823:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828763.380751:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828763.380762:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828763.380766:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828763.380782:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4167 00000100:00080000:0.0:1787828763.380796:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828763.380806:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828763.380865:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011712896/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828763.380908:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828763.380913:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828763.380916:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828763.380920:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4167 00000100:00080000:0.0:1787828763.380924:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828763.380928:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828763.380938:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828763.380955:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011713024/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828763.380955:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011712896:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828763.380984:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828763.380988:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828763.380990:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828763.380996:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4167 00000100:00080000:0.0:1787828763.381000:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828763.381003:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828763.381008:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828763.381021:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011713152/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828763.381030:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828763.381050:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828763.381052:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828763.381055:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4167 00000100:00080000:0.0:1787828763.381059:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828763.381062:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:3.0:1787828763.381068:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:0.0:1787828763.381070:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828763.381086:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011713280/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828763.381126:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828763/real 1787828763] req@ffff907d90847480 x1874670011712896/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828763.381143:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011712896/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828763.381154:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828763.381157:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828763.381161:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011712896:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828763.381201:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011713024:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828763.381222:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828763.381232:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828763/real 1787828763] req@ffff907d90845a40 x1874670011713024/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828763.381241:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011713024/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828763.381247:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828763.381249:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828763.381252:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011713024:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828763.381256:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011713152:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828763.381269:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828763.381275:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828763/real 1787828763] req@ffff907d90844f00 x1874670011713152/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828763.381288:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011713152/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828763.381294:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828763.381296:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828763.381297:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011713152:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828763.381301:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011713280:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828763.381311:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828763.381317:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828763/real 1787828763] req@ffff907d90846940 x1874670011713280/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828763.381324:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011713280/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828763.381329:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828763.381331:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828763.381333:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011713280:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828763.508691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828763.508814:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828764.532723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828764.532777:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828765.556717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828765.556738:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828766.580762:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828766.580788:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828767.604735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828767.604774:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828768.500715:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828768.500727:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828768.500732:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828768.500744:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4172 00000100:00080000:0.0:1787828768.500757:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828768.500766:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828768.500809:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011713408/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828768.500832:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828768.500835:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828768.500837:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828768.500840:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4172 00000100:00080000:0.0:1787828768.500842:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828768.500846:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828768.500856:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828768.500868:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011713408:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828768.500871:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011713536/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828768.500887:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828768.500891:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828768.500894:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828768.500899:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4172 00000100:00080000:0.0:1787828768.500903:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828768.500908:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828768.500915:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828768.500932:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011713664/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828768.500940:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1787828768.500945:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787828768.500946:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828768.500951:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828768.500956:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4172 00000100:00080000:0.0:1787828768.500960:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828768.500964:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828768.500970:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828768.500985:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011713792/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828768.500993:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828768/real 1787828768] req@ffff907d90846d00 x1874670011713408/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828768.501020:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011713408/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828768.501038:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828768.501043:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828768.501049:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011713408:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828768.501171:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011713536:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828768.501201:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828768.501217:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828768/real 1787828768] req@ffff907d90845680 x1874670011713536/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828768.501360:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011713536/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828768.501372:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828768.501377:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828768.501382:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011713536:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828768.501434:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011713664:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828768.501466:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828768.501481:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828768/real 1787828768] req@ffff907d90844780 x1874670011713664/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828768.501495:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011713664/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828768.501511:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828768.501515:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828768.501519:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011713664:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828768.501526:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011713792:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828768.501548:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828768.501561:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828768/real 1787828768] req@ffff907d908443c0 x1874670011713792/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828768.501574:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011713792/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828768.501582:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828768.501586:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828768.501591:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011713792:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828768.628708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828768.628780:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828769.652703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828769.652763:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828770.676689:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828770.676733:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828771.700680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828771.700762:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828772.724707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828772.724740:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828773.620703:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828773.620714:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828773.620718:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828773.620753:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4177 00000100:00080000:3.0:1787828773.620762:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828773.620768:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828773.620799:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011713920/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828773.620832:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828773.620835:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828773.620837:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828773.620839:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4177 00000100:00080000:3.0:1787828773.620842:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828773.620844:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828773.620851:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828773.620860:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011714048/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828773.620868:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828773.620873:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011713920:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828773.620873:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828773.620877:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828773.620884:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4177 00000100:00080000:3.0:1787828773.620886:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828773.620889:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828773.620892:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828773.620907:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011714176/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828773.620914:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828773.620917:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828773.620918:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828773.620921:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4177 00000100:00080000:3.0:1787828773.620923:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828773.620925:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828773.620928:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828773.620936:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011714304/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828773.620957:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828773.621008:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828773/real 1787828773] req@ffff907da562c000 x1874670011713920/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828773.621028:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011713920/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828773.621038:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828773.621041:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828773.621044:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011713920:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828773.621068:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011714048:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828773.621106:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828773.621122:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828773/real 1787828773] req@ffff907da562e940 x1874670011714048/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828773.621133:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011714048/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828773.621150:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828773.621152:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828773.621155:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011714048:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828773.621160:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011714176:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828773.621178:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828773.621189:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828773/real 1787828773] req@ffff907da562c3c0 x1874670011714176/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828773.621202:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011714176/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828773.621212:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828773.621215:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828773.621219:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011714176:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828773.621226:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011714304:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828773.621246:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828773.621258:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828773/real 1787828773] req@ffff907da562f0c0 x1874670011714304/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828773.621272:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011714304/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828773.621280:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828773.621282:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828773.621286:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011714304:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828773.748701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828773.748771:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828774.772724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828774.772769:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828775.796688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828775.796739:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828776.820739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828776.820779:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828777.844702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828777.844805:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828778.740697:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828778.740708:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828778.740713:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828778.740723:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4182 00000100:00080000:3.0:1787828778.740735:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828778.740742:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828778.740782:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011714432/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828778.740807:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828778.740811:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828778.740812:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828778.740815:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4182 00000100:00080000:3.0:1787828778.740818:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828778.740821:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828778.740831:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828778.740842:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011714560/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828778.740857:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828778.740861:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828778.740866:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828778.740868:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011714432:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828778.740874:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4182 00000100:00080000:3.0:1787828778.740886:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828778.740892:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828778.740896:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828778.740923:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011714688/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828778.740931:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828778.740934:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828778.740935:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828778.740940:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4182 00000100:00080000:3.0:1787828778.740946:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828778.740949:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828778.740956:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828778.740965:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011714816/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828778.741011:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011714560:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828778.741040:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011714688:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828778.741069:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011714816:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828778.868775:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828778.868776:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828779.892709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828779.892772:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828780.916724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828780.916766:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787828781.812820:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787828781.812862:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787828781.812879:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828781.812915:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828781.812925:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828781.812941:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:0.0:1787828781.812969:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828778/real 1787828781] req@ffff907da562f840 x1874670011714560/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828781.812997:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011714560/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828781.813010:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828781.813015:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828781.813020:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011714560:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828781.813028:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828778/real 1787828781] req@ffff907da562e1c0 x1874670011714688/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828781.813056:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011714688/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828781.813064:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828781.813068:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828781.813071:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011714688:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828781.813079:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828778/real 1787828781] req@ffff907da562cb40 x1874670011714816/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828781.813091:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011714816/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828781.813099:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828781.813103:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828781.813106:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011714816:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828781.813113:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828778/real 1787828781] req@ffff907da562da40 x1874670011714432/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828781.813125:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011714432/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828781.813132:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828781.813135:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828781.813138:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011714432:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828781.940766:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828781.940833:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828782.964731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828782.964789:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828783.860750:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828783.860761:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828783.860766:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828783.860777:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4187 00000100:00080000:0.0:1787828783.860790:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828783.860797:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828783.860837:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011714944/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828783.860883:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828783.860886:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828783.860888:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828783.860891:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4187 00000100:00080000:0.0:1787828783.860894:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828783.860896:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828783.860906:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828783.860916:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011715072/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828783.860924:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828783.860929:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1787828783.860930:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011714944:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828783.860932:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828783.860942:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4187 00000100:00080000:0.0:1787828783.860949:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828783.860954:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828783.860976:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828783.860991:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011715200/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828783.861003:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828783.861008:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828783.861010:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828783.861015:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4187 00000100:00080000:0.0:1787828783.861019:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828783.861039:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828783.861045:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828783.861059:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011715328/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828783.861065:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828783.861147:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828783/real 1787828783] req@ffff907d90844b40 x1874670011714944/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828783.861190:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011714944/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828783.861204:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828783.861211:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828783.861215:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011714944:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828783.861251:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011715072:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828783.861283:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828783.861299:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828783/real 1787828783] req@ffff907d90845e00 x1874670011715072/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828783.861313:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011715072/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828783.861322:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828783.861326:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828783.861329:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011715072:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828783.861336:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011715200:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828783.861360:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828783.861373:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828783/real 1787828783] req@ffff907d90844000 x1874670011715200/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828783.861394:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011715200/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828783.861404:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828783.861407:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828783.861411:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011715200:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828783.861417:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011715328:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828783.861437:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828783.861449:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828783/real 1787828783] req@ffff907d908452c0 x1874670011715328/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828783.861462:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011715328/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828783.861471:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828783.861475:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828783.861487:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011715328:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828783.988666:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828783.988738:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828785.012756:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828785.012831:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828786.036688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828786.036748:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828787.060690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828787.060710:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828788.084691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828788.084771:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828788.980764:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828788.980776:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828788.980781:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828788.980792:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4192 00000100:00080000:3.0:1787828788.980804:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828788.980812:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828788.980853:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011715456/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828788.980882:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828788.980885:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828788.980887:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828788.980891:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4192 00000100:00080000:3.0:1787828788.980895:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828788.980898:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828788.980907:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828788.980916:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011715456:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828788.980922:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011715584/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828788.980931:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828788.980934:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828788.980936:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828788.980939:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4192 00000100:00080000:3.0:1787828788.980941:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828788.980944:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828788.980947:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828788.980955:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011715712/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828788.980962:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828788.980965:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828788.980967:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828788.980969:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4192 00000100:00080000:3.0:1787828788.980972:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828788.980974:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:0.0:1787828788.980978:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:3.0:1787828788.980980:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828788.981003:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011715840/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828788.981033:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828788/real 1787828788] req@ffff907da562de00 x1874670011715456/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828788.981053:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011715456/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828788.981064:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828788.981068:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828788.981073:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011715456:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828788.981096:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011715584:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828788.981122:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828788.981133:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828788/real 1787828788] req@ffff907da562da40 x1874670011715584/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828788.981141:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011715584/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828788.981147:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828788.981150:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828788.981152:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011715584:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828788.981156:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011715712:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828788.981169:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828788.981175:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828788/real 1787828788] req@ffff907da562cb40 x1874670011715712/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828788.981183:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011715712/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828788.981188:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828788.981190:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828788.981191:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011715712:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828788.981195:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011715840:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828788.981205:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828788.981210:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828788/real 1787828788] req@ffff907da562e1c0 x1874670011715840/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828788.981217:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011715840/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828788.981222:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828788.981224:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828788.981226:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011715840:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828789.108710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828789.108742:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828790.132740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828790.132810:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828791.156755:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828791.156814:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828792.180749:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828792.180814:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828793.204717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828793.204757:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828794.100697:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828794.100704:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828794.100708:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828794.100716:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4197 00000100:00080000:3.0:1787828794.100726:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828794.100731:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828794.100761:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011715968/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828794.100785:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828794.100788:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828794.100789:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828794.100792:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4197 00000100:00080000:3.0:1787828794.100794:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828794.100797:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828794.100805:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828794.100814:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011716096/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828794.100820:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828794.100823:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828794.100824:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828794.100826:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4197 00000100:00080000:3.0:1787828794.100828:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828794.100830:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828794.100832:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828794.100837:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011716224/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828794.100842:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828794.100844:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828794.100845:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828794.100848:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4197 00000100:00080000:3.0:1787828794.100850:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828794.100852:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828794.100855:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828794.100861:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011716352/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828794.100881:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011715968:192.168.203.104@tcp:250:kworker.0 00000800:00000100:0.0:1787828794.100937:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828794.100993:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828794/real 1787828794] req@ffff907da562f840 x1874670011715968/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828794.101011:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011715968/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828794.101022:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828794.101026:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828794.101031:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011715968:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828794.101038:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011716096:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828794.101062:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828794.101077:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828794/real 1787828794] req@ffff907da562f0c0 x1874670011716096/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828794.101091:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011716096/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828794.101100:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828794.101103:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828794.101107:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011716096:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828794.101115:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011716224:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828794.101138:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828794.101150:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828794/real 1787828794] req@ffff907da562c3c0 x1874670011716224/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828794.101170:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011716224/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828794.101178:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828794.101181:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828794.101184:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011716224:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828794.101190:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011716352:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828794.101210:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828794.101221:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828794/real 1787828794] req@ffff907da562e940 x1874670011716352/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828794.101233:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011716352/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828794.101241:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828794.101244:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828794.101248:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011716352:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828794.228743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828794.228802:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828795.252736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828795.252801:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828796.276717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828796.276740:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828797.300702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828797.300760:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828798.324753:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828798.324833:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828799.220737:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828799.220754:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828799.220759:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828799.220774:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4202 00000100:00080000:3.0:1787828799.220789:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828799.220798:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828799.220843:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011716480/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828799.220873:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828799.220876:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828799.220878:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828799.220881:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4202 00000100:00080000:3.0:1787828799.220888:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828799.220890:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828799.220900:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828799.220911:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011716608/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828799.220930:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828799.220933:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828799.220935:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828799.220938:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4202 00000100:00080000:3.0:1787828799.220941:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828799.220945:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828799.220950:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828799.220957:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011716480:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828799.220962:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011716736/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828799.220973:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828799.220975:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828799.220976:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828799.220978:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4202 00000100:00080000:3.0:1787828799.220980:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828799.220982:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828799.220985:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828799.220995:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011716864/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828799.221052:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828799.221103:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828799/real 1787828799] req@ffff907da562c000 x1874670011716480/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828799.221123:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011716480/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828799.221135:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828799.221140:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828799.221144:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011716480:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828799.221152:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011716608:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828799.221175:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828799.221188:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828799/real 1787828799] req@ffff907da562fc00 x1874670011716608/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828799.221206:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011716608/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828799.221215:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828799.221218:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828799.221221:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011716608:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828799.221251:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011716736:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828799.221271:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828799.221282:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828799/real 1787828799] req@ffff907da562d2c0 x1874670011716736/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828799.221294:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011716736/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828799.221302:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828799.221305:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828799.221309:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011716736:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828799.221315:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011716864:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828799.221331:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828799.221341:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828799/real 1787828799] req@ffff907da562e580 x1874670011716864/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828799.221353:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011716864/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828799.221362:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828799.221365:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828799.221368:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011716864:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828799.348692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828799.348786:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828800.372717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828800.372766:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828801.396710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828801.396782:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828802.420730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828802.420824:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828803.444690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828803.444717:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828804.340741:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828804.340748:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828804.340752:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828804.340760:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4208 00000100:00080000:0.0:1787828804.340769:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828804.340776:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828804.340825:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011716992/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828804.340850:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828804.340854:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828804.340856:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828804.340859:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4208 00000100:00080000:0.0:1787828804.340863:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828804.340868:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828804.340892:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828804.340906:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011717120/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828804.340918:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011716992:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828804.340920:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828804.340928:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828804.340932:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828804.340953:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4208 00000100:00080000:0.0:1787828804.340956:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828804.340960:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828804.340965:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828804.340974:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011717248/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828804.340981:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828804.340983:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828804.340984:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828804.340987:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4208 00000100:00080000:0.0:1787828804.340990:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828804.340992:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828804.340995:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828804.341003:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011717376/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828804.341105:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828804.341310:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828804/real 1787828804] req@ffff907d90847c00 x1874670011716992/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828804.341334:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011716992/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828804.341348:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828804.341355:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828804.341360:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011716992:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828804.341388:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011717120:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828804.341417:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828804.341433:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828804/real 1787828804] req@ffff907d90846580 x1874670011717120/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828804.341449:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011717120/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828804.341459:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828804.341462:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828804.341467:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011717120:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828804.341474:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011717248:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828804.341499:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828804.341527:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828804/real 1787828804] req@ffff907d908470c0 x1874670011717248/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828804.341542:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011717248/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828804.341551:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828804.341555:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828804.341558:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011717248:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828804.341565:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011717376:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828804.341588:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828804.341602:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828804/real 1787828804] req@ffff907d908452c0 x1874670011717376/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828804.341652:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011717376/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828804.341662:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828804.341667:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828804.341674:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011717376:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828804.468700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828804.468764:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828805.492764:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828805.492898:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828806.516741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828806.516830:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828807.540745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828807.540790:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828808.564815:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828808.564938:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828809.460764:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828809.460781:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828809.460788:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828809.460804:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4213 00000100:00080000:3.0:1787828809.460822:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828809.460833:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828809.460921:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011717504/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828809.460966:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828809.460972:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828809.460975:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828809.460981:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4213 00000100:00080000:3.0:1787828809.460989:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828809.460996:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828809.461013:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828809.461030:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011717504:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828809.461034:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011717632/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828809.461058:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828809.461063:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828809.461065:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828809.461071:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4213 00000100:00080000:3.0:1787828809.461075:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828809.461080:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828809.461086:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828809.461103:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011717760/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828809.461115:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828809.461120:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828809.461122:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828809.461127:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4213 00000100:00080000:3.0:1787828809.461132:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828809.461137:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828809.461142:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787828809.461153:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828809.461161:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011717888/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828809.461227:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828809/real 1787828809] req@ffff907da562f480 x1874670011717504/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828809.461255:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011717504/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828809.461291:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828809.461302:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828809.461310:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011717504:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828809.461358:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011717632:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828809.461428:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828809.461456:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828809/real 1787828809] req@ffff907da562c780 x1874670011717632/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828809.461481:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011717632/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828809.461497:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828809.461504:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828809.461511:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011717632:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828809.461524:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011717760:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828809.461587:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828809.461650:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828809/real 1787828809] req@ffff907da562ed00 x1874670011717760/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828809.461688:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011717760/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828809.461700:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828809.461704:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828809.461709:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011717760:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828809.461718:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011717888:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828809.461751:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828809.461765:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828809/real 1787828809] req@ffff907da562cf00 x1874670011717888/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828809.461780:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011717888/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828809.461791:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828809.461795:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828809.461799:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011717888:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828809.588760:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828809.588867:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828810.612687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828810.612709:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828811.636739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828811.636874:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828812.660671:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828812.660910:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828813.684735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828813.684933:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828814.580841:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828814.580855:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828814.580861:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828814.580876:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4218 00000100:00080000:1.0:1787828814.580891:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828814.580901:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828814.580958:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011718016/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828814.580994:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828814.580999:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828814.581001:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828814.581013:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4218 00000100:00080000:1.0:1787828814.581017:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828814.581021:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828814.581035:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828814.581051:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011718144/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828814.581065:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828814.581067:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011718016:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828814.581072:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828814.581079:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828814.581086:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4218 00000100:00080000:1.0:1787828814.581089:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828814.581093:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828814.581098:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828814.581113:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011718272/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828814.581123:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828814.581127:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828814.581130:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828814.581135:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4218 00000100:00080000:1.0:1787828814.581150:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828814.581155:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828814.581160:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787828814.581171:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828814.581177:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011718400/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828814.581235:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828814/real 1787828814] req@ffff907d908a21c0 x1874670011718016/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828814.581258:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011718016/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828814.581273:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828814.581278:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828814.581283:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011718016:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828814.581312:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011718144:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828814.581337:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828814.581350:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828814/real 1787828814] req@ffff907d908a2580 x1874670011718144/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828814.581363:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011718144/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828814.581372:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828814.581376:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828814.581379:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011718144:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828814.581385:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011718272:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828814.581402:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828814.581412:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828814/real 1787828814] req@ffff907d908a2940 x1874670011718272/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828814.581423:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011718272/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828814.581431:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828814.581434:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828814.581437:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011718272:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828814.581443:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011718400:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828814.581459:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828814.581467:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828814/real 1787828814] req@ffff907d908a0000 x1874670011718400/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828814.581479:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011718400/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828814.581487:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828814.581489:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828814.581492:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011718400:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828814.708847:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828814.708901:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828815.732798:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828815.732924:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828816.756714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828816.756745:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828817.780781:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828817.780860:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828818.804699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828818.804773:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828819.700796:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828819.700807:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828819.700812:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828819.700824:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4223 00000100:00080000:1.0:1787828819.700837:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828819.700845:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828819.700889:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011718528/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828819.700917:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828819.700920:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828819.700921:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828819.700924:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4223 00000100:00080000:1.0:1787828819.700926:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828819.700929:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828819.700939:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828819.700951:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011718656/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828819.700959:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828819.700961:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828819.700963:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828819.700966:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4223 00000100:00080000:1.0:1787828819.700968:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828819.700971:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828819.700974:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828819.700986:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011718528:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828819.700987:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011718784/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828819.701012:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828819.701017:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828819.701019:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828819.701025:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4223 00000100:00080000:1.0:1787828819.701030:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828819.701037:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828819.701043:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828819.701059:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011718912/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828819.701074:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828819.701122:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828819/real 1787828819] req@ffff907d908a2d00 x1874670011718528/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828819.701139:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011718528/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828819.701149:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828819.701152:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828819.701156:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011718528:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828819.701163:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011718656:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828819.701193:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828819.701210:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828819/real 1787828819] req@ffff907d908a1e00 x1874670011718656/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828819.701227:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011718656/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828819.701239:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828819.701244:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828819.701247:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011718656:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828819.701273:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011718784:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828819.701316:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828819.701327:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828819/real 1787828819] req@ffff907d908a0000 x1874670011718784/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828819.701337:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011718784/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828819.701344:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828819.701347:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828819.701350:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011718784:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828819.701355:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011718912:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828819.701369:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828819.701376:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828819/real 1787828819] req@ffff907d908a2940 x1874670011718912/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828819.701397:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011718912/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828819.701404:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828819.701407:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828819.701409:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011718912:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828819.828741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828819.828767:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828820.852831:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828820.852927:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787828821.876828:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828821.876887:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828822.900795:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828822.900840:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828823.924710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828823.924774:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828824.820803:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828824.820820:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828824.820827:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828824.820843:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4228 00000100:00080000:2.0:1787828824.820860:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828824.820870:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828824.820924:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011719040/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828824.820960:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828824.820964:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828824.820966:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828824.820971:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4228 00000100:00080000:2.0:1787828824.820976:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828824.820981:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828824.820994:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828824.821013:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011719168/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828824.821025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828824.821032:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787828824.821034:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011719040:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828824.821035:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828824.821048:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4228 00000100:00080000:2.0:1787828824.821052:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828824.821058:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828824.821068:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828824.821085:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011719296/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828824.821101:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828824.821104:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828824.821106:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828824.821110:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4228 00000100:00080000:2.0:1787828824.821115:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828824.821119:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828824.821125:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828824.821138:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011719424/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828824.821169:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828824.821257:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828824/real 1787828824] req@ffff907d9114a580 x1874670011719040/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828824.821274:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011719040/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828824.821283:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828824.821286:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828824.821289:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011719040:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828824.821309:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011719168:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828824.821328:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828824.821338:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828824/real 1787828824] req@ffff907d9114ad00 x1874670011719168/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828824.821346:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011719168/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828824.821352:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828824.821355:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828824.821357:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011719168:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828824.821361:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011719296:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828824.821378:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828824.821388:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828824/real 1787828824] req@ffff907d91149a40 x1874670011719296/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828824.821401:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011719296/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828824.821410:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828824.821414:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828824.821418:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011719296:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828824.821425:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011719424:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828824.821456:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828824.821469:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828824/real 1787828824] req@ffff907d9114b840 x1874670011719424/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828824.821481:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011719424/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828824.821490:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828824.821493:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828824.821498:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011719424:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828824.948681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828824.948707:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828825.972693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828825.972717:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828826.996681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828826.996734:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828828.020778:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828828.020804:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828829.044781:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828829.044857:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828829.940834:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828829.940851:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828829.940858:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828829.940875:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4233 00000100:00080000:0.0:1787828829.940894:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828829.940905:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828829.940981:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011719552/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828829.941024:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828829.941045:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828829.941048:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828829.941055:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4233 00000100:00080000:0.0:1787828829.941061:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828829.941067:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828829.941082:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828829.941104:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011719680/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828829.941124:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828829.941124:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011719552:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828829.941138:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828829.941140:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828829.941148:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4233 00000100:00080000:0.0:1787828829.941155:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828829.941162:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828829.941171:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828829.941192:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011719808/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828829.941207:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1787828829.941216:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787828829.941217:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828829.941222:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828829.941230:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4233 00000100:00080000:0.0:1787828829.941238:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828829.941244:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828829.941252:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828829.941287:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011719936/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828829.941311:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828829/real 1787828829] req@ffff907d90844000 x1874670011719552/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828829.941341:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011719552/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828829.941357:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828829.941379:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828829.941386:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011719552:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828829.941421:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011719680:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828829.941453:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828829.941467:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828829/real 1787828829] req@ffff907d90845e00 x1874670011719680/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828829.941487:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011719680/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828829.941495:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828829.941498:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828829.941501:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011719680:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828829.941507:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011719808:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828829.941541:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828829.941550:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828829/real 1787828829] req@ffff907d90844b40 x1874670011719808/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828829.941561:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011719808/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828829.941568:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828829.941570:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828829.941572:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011719808:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828829.941577:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011719936:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828829.941590:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828829.941598:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828829/real 1787828829] req@ffff907d908443c0 x1874670011719936/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828829.941607:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011719936/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828829.941631:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828829.941634:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828829.941667:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011719936:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828830.068745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828830.068801:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828831.092711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828831.092784:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828832.116700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828832.116769:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828833.140687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828833.140753:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828834.164690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828834.164778:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828835.060732:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828835.060743:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828835.060747:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828835.060757:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4238 00000100:00080000:2.0:1787828835.060766:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828835.060772:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828835.060804:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011720064/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828835.060825:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828835.060827:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828835.060829:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828835.060832:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4238 00000100:00080000:2.0:1787828835.060846:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828835.060850:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828835.060858:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828835.060868:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011720064:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828835.060869:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011720192/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828835.060882:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828835.060885:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828835.060887:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828835.060889:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4238 00000100:00080000:2.0:1787828835.060892:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828835.060894:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828835.060897:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828835.060907:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011720320/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828835.060913:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828835.060916:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828835.060917:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828835.060919:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4238 00000100:00080000:2.0:1787828835.060921:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828835.060924:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:3.0:1787828835.060928:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:2.0:1787828835.060931:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828835.060942:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011720448/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828835.060969:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828835/real 1787828835] req@ffff907d9114b480 x1874670011720064/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828835.061005:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011720064/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828835.061017:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828835.061022:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828835.061026:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011720064:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828835.061066:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011720192:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828835.061085:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828835.061095:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828835/real 1787828835] req@ffff907d91149680 x1874670011720192/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828835.061108:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011720192/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828835.061118:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828835.061122:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828835.061126:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011720192:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828835.061132:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011720320:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828835.061155:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828835.061166:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828835/real 1787828835] req@ffff907d9114bc00 x1874670011720320/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828835.061175:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011720320/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828835.061181:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828835.061183:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828835.061185:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011720320:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828835.061189:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011720448:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828835.061202:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828835.061209:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828835/real 1787828835] req@ffff907d911483c0 x1874670011720448/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828835.061216:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011720448/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828835.061221:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828835.061223:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828835.061225:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011720448:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828835.188726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828835.188814:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828836.212691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828836.212739:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828837.236716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828837.236732:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828838.260724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828838.260784:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828839.284718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828839.284746:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828840.180716:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828840.180725:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828840.180729:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828840.180737:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4243 00000100:00080000:0.0:1787828840.180746:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828840.180753:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828840.180784:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011720576/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828840.180810:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828840.180813:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828840.180815:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828840.180818:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4243 00000100:00080000:0.0:1787828840.180820:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828840.180823:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828840.180832:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828840.180845:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011720704/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828840.180852:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011720576:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828840.180855:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828840.180863:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828840.180865:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828840.180870:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4243 00000100:00080000:0.0:1787828840.180875:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828840.180881:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828840.180887:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828840.180902:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011720832/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828840.180914:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787828840.180916:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828840.180921:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828840.180924:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828840.180930:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4243 00000100:00080000:0.0:1787828840.180935:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828840.180939:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828840.180945:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828840.180949:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828840/real 1787828840] req@ffff907d90844780 x1874670011720576/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828840.180960:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011720960/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828840.180966:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011720576/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828840.180988:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828840.180993:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828840.180999:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011720576:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828840.181024:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011720704:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828840.181046:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828840.181066:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828840/real 1787828840] req@ffff907d90845680 x1874670011720704/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828840.181075:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011720704/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828840.181084:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828840.181087:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828840.181090:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011720704:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828840.181097:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011720832:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828840.181119:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828840.181132:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828840/real 1787828840] req@ffff907d90846d00 x1874670011720832/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828840.181145:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011720832/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828840.181151:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828840.181153:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828840.181155:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011720832:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828840.181160:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011720960:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828840.181175:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828840.181183:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828840/real 1787828840] req@ffff907d90846940 x1874670011720960/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828840.181190:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011720960/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828840.181197:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828840.181199:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828840.181201:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011720960:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828840.308736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828840.308788:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828841.332694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828841.332752:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828842.356706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828842.356770:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828843.380709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828843.380767:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828844.404665:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828844.404747:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828845.300699:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828845.300710:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828845.300714:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828845.300726:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4249 00000100:00080000:0.0:1787828845.300738:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828845.300745:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828845.300788:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011721088/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828845.300814:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828845.300817:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828845.300819:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828845.300822:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4249 00000100:00080000:0.0:1787828845.300824:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828845.300826:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828845.300835:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828845.300845:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011721216/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828845.300853:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828845.300858:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828845.300860:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828845.300863:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4249 00000100:00080000:0.0:1787828845.300865:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828845.300869:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828845.300873:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828845.300881:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011721344/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828845.300889:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011721088:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828845.300891:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828845.300897:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828845.300902:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828845.300913:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4249 00000100:00080000:0.0:1787828845.300916:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828845.300919:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828845.300922:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828845.300933:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011721472/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828845.301015:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011721216:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828845.301056:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011721344:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828845.301083:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011721472:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828845.428706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828845.428728:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828846.452672:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828846.452764:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828847.476721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828847.476758:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787828848.372859:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787828848.372909:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787828848.372929:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828848.372979:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828848.372991:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828848.373005:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:2.0:1787828848.373036:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828845/real 1787828848] req@ffff907d90847480 x1874670011721344/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828848.373067:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011721344/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828848.373082:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828848.373088:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828848.373094:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011721344:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828848.373103:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828845/real 1787828848] req@ffff907d90846940 x1874670011721472/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828848.373117:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011721472/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828848.373125:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828848.373129:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828848.373133:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011721472:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828848.373140:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828845/real 1787828848] req@ffff907d90844f00 x1874670011721088/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828848.373154:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011721088/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828848.373163:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828848.373166:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828848.373170:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011721088:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828848.373178:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828845/real 1787828848] req@ffff907d90845a40 x1874670011721216/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828848.373191:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011721216/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828848.373198:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828848.373202:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828848.373205:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011721216:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828848.500672:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828848.500782:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828849.524699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828849.524717:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828850.420766:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828850.420780:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828850.420785:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828850.420798:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4254 00000100:00080000:1.0:1787828850.420812:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828850.420823:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828850.420867:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011721600/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828850.420906:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828850.420910:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828850.420913:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828850.420918:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4254 00000100:00080000:1.0:1787828850.420922:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828850.420927:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828850.420940:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828850.420956:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011721728/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828850.420965:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011721600:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828850.420972:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828850.420977:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828850.420979:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828850.420984:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4254 00000100:00080000:1.0:1787828850.420988:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828850.420993:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828850.421001:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828850.421015:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011721856/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828850.421026:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828850.421030:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828850.421032:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828850.421036:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4254 00000100:00080000:1.0:1787828850.421039:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828850.421044:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828850.421049:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787828850.421062:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828850.421064:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011721984/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828850.421116:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828850/real 1787828850] req@ffff907d908a2580 x1874670011721600/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828850.421139:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011721600/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828850.421153:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828850.421158:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828850.421163:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011721600:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828850.421193:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011721728:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828850.421218:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828850.421232:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828850/real 1787828850] req@ffff907d908a21c0 x1874670011721728/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828850.421253:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011721728/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828850.421263:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828850.421267:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828850.421270:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011721728:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828850.421277:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011721856:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828850.421301:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828850.421314:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828850/real 1787828850] req@ffff907d908a3840 x1874670011721856/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828850.421327:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011721856/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828850.421344:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828850.421348:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828850.421352:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011721856:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828850.421359:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011721984:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828850.421381:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828850.421393:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828850/real 1787828850] req@ffff907d908a1680 x1874670011721984/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828850.421406:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011721984/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828850.421422:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828850.421426:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828850.421430:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011721984:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828850.548730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828850.548814:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828851.572693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828851.572740:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828852.596695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828852.596726:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828853.620754:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828853.620803:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828854.644707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828854.644765:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828855.540722:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828855.540733:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828855.540737:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828855.540748:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4259 00000100:00080000:1.0:1787828855.540760:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828855.540767:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828855.540827:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011722112/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828855.540860:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828855.540863:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828855.540865:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828855.540868:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4259 00000100:00080000:1.0:1787828855.540871:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828855.540876:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828855.540889:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828855.540908:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011722240/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828855.540922:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828855.540924:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011722112:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828855.540927:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828855.540936:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828855.540944:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4259 00000100:00080000:1.0:1787828855.540950:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828855.540958:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828855.540964:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828855.540983:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011722368/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828855.540991:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828855.540993:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828855.540995:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828855.540999:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4259 00000100:00080000:1.0:1787828855.541003:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828855.541010:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:2.0:1787828855.541011:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:1.0:1787828855.541021:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828855.541038:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011722496/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828855.541065:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828855/real 1787828855] req@ffff907d908a12c0 x1874670011722112/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828855.541082:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011722112/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828855.541091:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828855.541094:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828855.541097:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011722112:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828855.541127:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011722240:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828855.541152:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828855.541167:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828855/real 1787828855] req@ffff907d908a0f00 x1874670011722240/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828855.541181:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011722240/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828855.541190:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828855.541195:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828855.541208:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011722240:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828855.541217:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011722368:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828855.541240:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828855.541253:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828855/real 1787828855] req@ffff907d908a1680 x1874670011722368/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828855.541277:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011722368/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828855.541287:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828855.541289:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828855.541292:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011722368:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828855.541297:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011722496:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828855.541314:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828855.541323:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828855/real 1787828855] req@ffff907d908a3840 x1874670011722496/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828855.541330:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011722496/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828855.541338:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828855.541342:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828855.541346:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011722496:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828855.668700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828855.668791:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828856.692691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828856.692764:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828857.716715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828857.716825:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828858.740709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828858.740746:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828859.764755:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828859.764816:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828860.660721:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828860.660730:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828860.660733:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828860.660755:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4264 00000100:00080000:2.0:1787828860.660765:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828860.660771:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828860.660805:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011722624/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828860.660828:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828860.660831:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828860.660832:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828860.660836:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4264 00000100:00080000:2.0:1787828860.660838:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828860.660841:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828860.660849:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828860.660859:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011722752/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828860.660867:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828860.660871:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828860.660872:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828860.660875:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4264 00000100:00080000:2.0:1787828860.660877:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828860.660883:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828860.660883:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011722624:192.168.203.104@tcp:250:kworker.0 00000008:00100000:2.0:1787828860.660887:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828860.660898:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011722880/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828860.660906:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828860.660909:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828860.660911:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828860.660915:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4264 00000100:00080000:2.0:1787828860.660919:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828860.660923:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828860.660929:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828860.660943:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011723008/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828860.660959:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828860.661020:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828860/real 1787828860] req@ffff907d911483c0 x1874670011722624/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828860.661035:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011722624/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828860.661044:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828860.661047:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828860.661050:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011722624:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828860.661056:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011722752:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828860.661075:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828860.661084:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828860/real 1787828860] req@ffff907d9114bc00 x1874670011722752/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828860.661093:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011722752/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828860.661099:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828860.661102:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828860.661104:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011722752:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828860.661122:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011722880:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828860.661135:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828860.661142:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828860/real 1787828860] req@ffff907d91149680 x1874670011722880/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828860.661150:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011722880/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828860.661162:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828860.661164:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828860.661166:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011722880:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828860.661170:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011723008:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828860.661181:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828860.661187:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828860/real 1787828860] req@ffff907d9114b480 x1874670011723008/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828860.661195:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011723008/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828860.661200:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828860.661202:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828860.661204:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011723008:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828860.788746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828860.788803:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828861.812740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828861.812826:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828862.836704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828862.836817:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828863.860729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828863.860748:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828864.884682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828864.884755:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828865.780770:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828865.780781:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828865.780785:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828865.780796:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4269 00000100:00080000:2.0:1787828865.780808:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828865.780816:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828865.780861:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011723136/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828865.780910:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828865.780915:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828865.780918:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828865.780923:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4269 00000100:00080000:2.0:1787828865.780927:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828865.780931:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828865.780944:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828865.780959:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011723264/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828865.780972:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828865.780976:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828865.780978:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828865.780983:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4269 00000100:00080000:2.0:1787828865.780986:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828865.780991:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828865.780997:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828865.781012:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011723392/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828865.781017:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011723136:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828865.781028:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828865.781036:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828865.781038:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828865.781042:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4269 00000100:00080000:2.0:1787828865.781045:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828865.781048:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828865.781069:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828865.781082:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011723520/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828865.781159:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828865.781224:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828865/real 1787828865] req@ffff907d9114b840 x1874670011723136/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828865.781247:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011723136/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828865.781261:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828865.781267:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828865.781272:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011723136:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828865.781281:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011723264:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828865.781309:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828865.781325:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828865/real 1787828865] req@ffff907d91149a40 x1874670011723264/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828865.781364:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011723264/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828865.781379:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828865.781386:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828865.781410:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011723264:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828865.781448:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011723392:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828865.781502:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828865.781519:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828865/real 1787828865] req@ffff907d9114ad00 x1874670011723392/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828865.781536:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011723392/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828865.781548:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828865.781552:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828865.781556:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011723392:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828865.781565:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011723520:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828865.781589:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828865.781604:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828865/real 1787828865] req@ffff907d9114a580 x1874670011723520/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828865.781692:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011723520/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828865.781708:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828865.781730:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828865.781733:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011723520:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828865.908779:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828865.908825:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828866.932686:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828866.932770:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828867.956729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828867.956800:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828868.980693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828868.980808:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828870.004710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828870.004757:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828870.900735:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828870.900746:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828870.900751:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828870.900763:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4274 00000100:00080000:2.0:1787828870.900776:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828870.900785:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828870.900825:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011723648/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828870.900856:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828870.900861:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828870.900863:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828870.900868:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4274 00000100:00080000:2.0:1787828870.900872:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828870.900877:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828870.900888:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828870.900900:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011723648:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828870.900906:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011723776/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828870.900920:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828870.900924:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828870.900926:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828870.900930:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4274 00000100:00080000:2.0:1787828870.900936:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828870.900940:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828870.900946:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828870.900959:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011723904/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828870.900971:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787828870.900972:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828870.900978:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828870.900980:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828870.900986:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4274 00000100:00080000:2.0:1787828870.900994:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828870.900998:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828870.901004:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828870.901011:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828870/real 1787828870] req@ffff907d911492c0 x1874670011723648/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828870.901021:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011724032/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828870.901031:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011723648/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828870.901043:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828870.901056:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828870.901060:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011723648:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828870.901079:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011723776:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828870.901097:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828870.901106:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828870/real 1787828870] req@ffff907d91148f00 x1874670011723776/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828870.901115:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011723776/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828870.901120:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828870.901122:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828870.901125:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011723776:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828870.901129:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011723904:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828870.901140:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828870.901146:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828870/real 1787828870] req@ffff907d9114a940 x1874670011723904/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828870.901162:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011723904/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828870.901167:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828870.901169:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828870.901172:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011723904:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828870.901175:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011724032:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828870.901186:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828870.901193:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828870/real 1787828870] req@ffff907d91148b40 x1874670011724032/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828870.901200:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011724032/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828870.901205:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828870.901207:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828870.901214:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011724032:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828871.028692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828871.028765:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828872.052716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828872.052800:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828873.076732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828873.076754:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828874.100729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828874.100763:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828875.124716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828875.124763:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828876.020740:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828876.020751:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828876.020756:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828876.020768:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4279 00000100:00080000:1.0:1787828876.020782:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828876.020794:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828876.020844:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011724160/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828876.020872:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828876.020875:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828876.020877:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828876.020881:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4279 00000100:00080000:1.0:1787828876.020886:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828876.020890:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828876.020901:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828876.020907:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011724160:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828876.020916:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011724288/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828876.020926:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828876.020928:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828876.020930:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828876.020934:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4279 00000100:00080000:1.0:1787828876.020937:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828876.020940:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828876.020943:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828876.020952:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011724416/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828876.020959:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828876.020962:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828876.020964:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828876.020968:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4279 00000100:00080000:1.0:1787828876.020972:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828876.020974:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828876.020978:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787828876.020980:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828876.020993:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011724544/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828876.021027:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828876/real 1787828876] req@ffff907d908a21c0 x1874670011724160/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828876.021048:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011724160/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828876.021060:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828876.021065:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828876.021070:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011724160:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828876.021106:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011724288:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828876.021133:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828876.021147:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828876/real 1787828876] req@ffff907d908a2580 x1874670011724288/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828876.021161:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011724288/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828876.021177:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828876.021181:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828876.021184:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011724288:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828876.021190:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011724416:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828876.021211:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828876.021219:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828876/real 1787828876] req@ffff907d908a2940 x1874670011724416/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828876.021234:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011724416/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828876.021240:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828876.021242:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828876.021245:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011724416:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828876.021249:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011724544:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828876.021261:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828876.021268:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828876/real 1787828876] req@ffff907d908a0000 x1874670011724544/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828876.021275:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011724544/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828876.021281:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828876.021282:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828876.021284:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011724544:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828876.148723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828876.148784:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828877.172699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828877.172753:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828878.196707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828878.196759:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828879.220693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828879.220783:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828880.244763:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828880.244870:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828881.140773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828881.140781:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828881.140785:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828881.140794:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4284 00000100:00080000:3.0:1787828881.140803:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828881.140809:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828881.140842:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011724672/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828881.140874:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828881.140877:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828881.140879:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828881.140899:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4284 00000100:00080000:3.0:1787828881.140901:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828881.140904:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828881.140912:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828881.140924:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011724800/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828881.140933:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828881.140937:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011724672:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828881.140937:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828881.140942:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828881.140949:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4284 00000100:00080000:3.0:1787828881.140951:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828881.140953:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828881.140956:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828881.140966:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011724928/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828881.140973:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828881.140992:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828881.140993:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828881.140997:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4284 00000100:00080000:3.0:1787828881.141011:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828881.141014:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828881.141017:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787828881.141023:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828881.141037:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011725056/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828881.141073:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828881/real 1787828881] req@ffff907da562d680 x1874670011724672/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828881.141089:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011724672/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828881.141113:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828881.141115:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828881.141118:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011724672:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828881.141139:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011724800:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828881.141182:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828881.141196:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828881/real 1787828881] req@ffff907da562cf00 x1874670011724800/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828881.141210:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011724800/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828881.141216:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828881.141219:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828881.141221:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011724800:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828881.141244:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011724928:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828881.141263:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828881.141272:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828881/real 1787828881] req@ffff907da562ed00 x1874670011724928/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828881.141281:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011724928/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828881.141287:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828881.141290:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828881.141292:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011724928:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828881.141296:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011725056:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828881.141308:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828881.141316:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828881/real 1787828881] req@ffff907da562c780 x1874670011725056/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828881.141324:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011725056/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828881.141336:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828881.141338:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828881.141341:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011725056:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828881.268719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828881.268814:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828882.292720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828882.292748:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828883.316712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828883.316816:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828884.340748:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828884.340771:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787828885.364762:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828885.364763:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:3.0:1787828886.260768:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828886.260781:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828886.260786:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828886.260798:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4290 00000100:00080000:3.0:1787828886.260813:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828886.260820:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828886.260873:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011725184/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828886.260913:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828886.260918:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828886.260921:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828886.260926:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4290 00000100:00080000:3.0:1787828886.260930:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828886.260935:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828886.260947:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828886.260961:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011725312/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828886.260973:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828886.260977:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011725184:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828886.260978:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828886.260982:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828886.260991:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4290 00000100:00080000:3.0:1787828886.260994:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828886.260997:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828886.261002:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828886.261013:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011725440/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828886.261022:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828886.261035:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828886.261036:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828886.261040:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4290 00000100:00080000:3.0:1787828886.261051:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828886.261055:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828886.261059:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787828886.261064:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828886.261072:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011725568/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828886.261121:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828886/real 1787828886] req@ffff907da562f480 x1874670011725184/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828886.261155:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011725184/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828886.261164:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828886.261167:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828886.261170:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011725184:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828886.261212:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011725312:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828886.261233:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828886.261244:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828886/real 1787828886] req@ffff907da562e580 x1874670011725312/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828886.261254:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011725312/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828886.261278:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828886.261281:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828886.261284:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011725312:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828886.261290:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011725440:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828886.261307:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828886.261340:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828886/real 1787828886] req@ffff907da562d2c0 x1874670011725440/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828886.261393:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011725440/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828886.261404:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828886.261409:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828886.261413:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011725440:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828886.261421:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011725568:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828886.261447:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828886.261479:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828886/real 1787828886] req@ffff907da562fc00 x1874670011725568/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828886.261491:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011725568/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828886.261500:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828886.261503:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828886.261507:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011725568:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828886.388740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828886.388835:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828887.412705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828887.412763:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828888.436776:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828888.436817:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828889.460732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828889.460801:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828890.484686:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828890.484750:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828891.380754:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828891.380763:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828891.380768:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828891.380778:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4295 00000100:00080000:2.0:1787828891.380788:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828891.380796:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828891.380836:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011725696/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828891.380878:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828891.380888:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828891.380890:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828891.380895:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4295 00000100:00080000:2.0:1787828891.380899:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828891.380903:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828891.380912:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828891.380914:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011725696:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828891.380925:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011725824/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828891.380936:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828891.380938:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828891.380940:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828891.380944:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4295 00000100:00080000:2.0:1787828891.380962:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828891.380966:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828891.380970:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787828891.380972:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828891.380980:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011725952/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828891.380988:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828891.380991:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828891.380993:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828891.380997:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4295 00000100:00080000:2.0:1787828891.381000:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828891.381003:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828891.381007:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828891.381014:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828891/real 1787828891] req@ffff907d91148b40 x1874670011725696/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828891.381015:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011726080/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828891.381033:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011725696/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828891.381044:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828891.381047:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828891.381051:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011725696:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828891.381074:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011725824:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828891.381093:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828891.381121:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828891/real 1787828891] req@ffff907d9114a940 x1874670011725824/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828891.381132:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011725824/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828891.381139:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828891.381142:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828891.381145:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011725824:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828891.381150:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011725952:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828891.381181:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828891.381191:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828891/real 1787828891] req@ffff907d91148f00 x1874670011725952/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828891.381204:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011725952/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828891.381211:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828891.381214:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828891.381217:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011725952:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828891.381223:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011726080:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828891.381241:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828891.381252:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828891/real 1787828891] req@ffff907d911492c0 x1874670011726080/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828891.381262:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011726080/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828891.381270:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828891.381286:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828891.381290:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011726080:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828891.508762:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828891.508812:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828892.532688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828892.532738:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828893.556750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828893.556816:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828894.580691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828894.580724:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828895.604689:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828895.604714:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828896.500836:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828896.500847:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828896.500852:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828896.500862:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4300 00000100:00080000:3.0:1787828896.500876:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828896.500883:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828896.500926:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011726208/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828896.500953:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828896.500957:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828896.500960:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828896.500966:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4300 00000100:00080000:3.0:1787828896.500972:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828896.500979:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828896.501009:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828896.501010:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011726208:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828896.501032:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011726336/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828896.501043:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828896.501047:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828896.501048:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828896.501052:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4300 00000100:00080000:3.0:1787828896.501056:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828896.501060:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828896.501082:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828896.501094:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011726464/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828896.501103:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828896.501107:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828896.501108:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828896.501111:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4300 00000100:00080000:3.0:1787828896.501115:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787828896.501118:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787828896.501123:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828896.501144:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828896.501156:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011726592/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828896.501189:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828896/real 1787828896] req@ffff907da562c000 x1874670011726208/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828896.501226:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011726208/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828896.501257:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828896.501276:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828896.501280:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011726208:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828896.501310:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011726336:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828896.501348:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828896.501359:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828896/real 1787828896] req@ffff907da562e940 x1874670011726336/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828896.501369:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011726336/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828896.501376:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828896.501379:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828896.501381:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011726336:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828896.501385:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011726464:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828896.501398:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828896.501405:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828896/real 1787828896] req@ffff907da562c3c0 x1874670011726464/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828896.501413:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011726464/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828896.501420:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828896.501422:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828896.501424:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011726464:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828896.501428:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011726592:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828896.501439:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828896.501446:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828896/real 1787828896] req@ffff907da562f0c0 x1874670011726592/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828896.501454:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011726592/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828896.501460:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828896.501462:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828896.501464:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011726592:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828896.628721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828896.628769:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828897.652723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828897.652821:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828898.676784:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828898.676843:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828899.700706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828899.700797:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828900.724701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828900.724747:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828901.620746:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828901.620756:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828901.620761:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828901.620771:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4305 00000100:00080000:1.0:1787828901.620783:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828901.620790:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828901.620833:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011726720/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828901.620871:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828901.620873:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828901.620875:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828901.620878:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4305 00000100:00080000:1.0:1787828901.620880:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828901.620884:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828901.620894:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828901.620904:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011726848/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828901.620924:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828901.620928:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828901.620930:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828901.620935:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4305 00000100:00080000:1.0:1787828901.620938:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828901.620945:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828901.620947:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011726720:192.168.203.104@tcp:250:kworker.0 00000008:00100000:1.0:1787828901.620951:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828901.620968:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011726976/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828901.620980:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828901.620984:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828901.620986:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828901.620991:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4305 00000100:00080000:1.0:1787828901.620994:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828901.620997:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828901.621002:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828901.621012:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011727104/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828901.621032:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828901.621077:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828901/real 1787828901] req@ffff907d908a1e00 x1874670011726720/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828901.621092:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011726720/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828901.621103:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828901.621106:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828901.621109:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011726720:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828901.621116:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011726848:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828901.621133:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828901.621142:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828901/real 1787828901] req@ffff907d908a2d00 x1874670011726848/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828901.621151:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011726848/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828901.621157:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828901.621159:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828901.621172:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011726848:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828901.621195:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011726976:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828901.621226:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828901.621235:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828901/real 1787828901] req@ffff907d908a0000 x1874670011726976/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828901.621243:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011726976/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828901.621249:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828901.621252:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828901.621254:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011726976:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828901.621258:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011727104:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828901.621269:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828901.621275:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828901/real 1787828901] req@ffff907d908a2940 x1874670011727104/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828901.621284:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011727104/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828901.621289:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828901.621291:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828901.621293:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011727104:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828901.748775:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828901.748790:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787828902.772716:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828902.772753:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787828903.796754:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828903.796782:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828904.820736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828904.820792:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828905.844759:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828905.844826:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828906.740711:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828906.740729:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828906.740734:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828906.740745:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4310 00000100:00080000:0.0:1787828906.740758:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828906.740765:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828906.740813:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011727232/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828906.740839:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828906.740842:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828906.740844:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828906.740848:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4310 00000100:00080000:0.0:1787828906.740851:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828906.740854:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828906.740865:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828906.740876:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011727360/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828906.740886:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828906.740889:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828906.740891:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828906.740896:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4310 00000100:00080000:0.0:1787828906.740900:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828906.740905:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828906.740911:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828906.740924:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011727488/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828906.740937:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011727232:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828906.740938:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828906.740943:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828906.740947:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828906.740953:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4310 00000100:00080000:0.0:1787828906.740957:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828906.740962:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828906.740967:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828906.740982:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011727616/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828906.741030:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828906.741092:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828906/real 1787828906] req@ffff907d90846d00 x1874670011727232/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828906.741108:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011727232/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828906.741118:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828906.741121:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828906.741124:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011727232:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828906.741131:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011727360:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828906.741149:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828906.741157:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828906/real 1787828906] req@ffff907d90845680 x1874670011727360/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828906.741181:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011727360/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828906.741189:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828906.741202:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828906.741205:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011727360:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828906.741229:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011727488:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828906.741246:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828906.741253:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828906/real 1787828906] req@ffff907d90844780 x1874670011727488/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828906.741273:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011727488/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828906.741279:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828906.741281:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828906.741283:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011727488:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828906.741286:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011727616:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828906.741295:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828906.741301:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828906/real 1787828906] req@ffff907d908443c0 x1874670011727616/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828906.741308:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011727616/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828906.741319:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828906.741321:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828906.741323:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011727616:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828906.868773:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828906.868826:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828907.892765:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828907.892829:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828908.916765:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828908.916831:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828909.940699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828909.940726:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828910.964701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828910.964748:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828911.860748:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828911.860757:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828911.860762:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828911.860774:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4315 00000100:00080000:0.0:1787828911.860788:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828911.860796:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828911.860835:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011727744/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828911.860862:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828911.860866:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828911.860868:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828911.860874:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4315 00000100:00080000:0.0:1787828911.860880:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828911.860885:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828911.860897:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011727744:192.168.203.104@tcp:250:kworker.0 00000008:00100000:0.0:1787828911.860900:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828911.860924:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011727872/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828911.860939:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828911.860943:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828911.860945:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828911.860949:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4315 00000100:00080000:0.0:1787828911.860953:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828911.860958:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828911.860963:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828911.860974:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011728000/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828911.860981:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828911.860984:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828911.860986:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828911.860989:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4315 00000100:00080000:0.0:1787828911.860991:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828911.860993:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828911.860997:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828911.861006:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011728128/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828911.861095:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011727872:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828911.861260:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011728000:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828911.861283:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011728128:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828911.988696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828911.988752:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828913.012710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828913.012743:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787828914.036704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828914.036756:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787828914.932839:0:2337:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787828914.932879:0:2337:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787828914.932897:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828914.932938:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828914.932951:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787828914.932963:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:3.0:1787828914.932980:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828911/real 1787828914] req@ffff907d90845e00 x1874670011727872/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828914.933006:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011727872/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828914.933018:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828914.933023:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828914.933028:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011727872:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828914.933036:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828911/real 1787828914] req@ffff907d90844000 x1874670011728000/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828914.933049:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011728000/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828914.933058:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828914.933061:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828914.933065:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011728000:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828914.933072:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828911/real 1787828914] req@ffff907d908452c0 x1874670011728128/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828914.933086:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011728128/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828914.933094:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828914.933097:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828914.933101:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011728128:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828914.933108:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828911/real 1787828914] req@ffff907d90844b40 x1874670011727744/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828914.933136:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011727744/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828914.933145:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828914.933148:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828914.933151:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011727744:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828915.060796:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828915.060810:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787828916.084709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828916.084793:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828916.980686:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828916.980693:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828916.980697:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828916.980706:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4320 00000100:00080000:3.0:1787828916.980714:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828916.980720:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828916.980751:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011728256/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828916.980784:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828916.980788:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828916.980790:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828916.980795:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4320 00000100:00080000:3.0:1787828916.980799:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828916.980803:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828916.980813:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828916.980829:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011728384/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828916.980843:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828916.980845:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011728256:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828916.980846:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828916.980850:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828916.980856:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4320 00000100:00080000:3.0:1787828916.980861:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828916.980865:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828916.980872:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828916.980886:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011728512/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828916.980897:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828916.980900:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828916.980902:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828916.980906:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4320 00000800:00000100:2.0:1787828916.980912:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787828916.980914:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828916.980921:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828916.980927:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828916.980942:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011728640/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828916.980958:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828916/real 1787828916] req@ffff907da562f840 x1874670011728256/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828916.980982:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011728256/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828916.980995:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828916.981001:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828916.981005:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011728256:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828916.981028:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011728384:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828916.981053:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828916.981068:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828916/real 1787828916] req@ffff907da562e1c0 x1874670011728384/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828916.981079:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011728384/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828916.981087:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828916.981090:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828916.981093:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011728384:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828916.981118:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011728512:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828916.981139:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828916.981151:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828916/real 1787828916] req@ffff907da562cb40 x1874670011728512/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828916.981163:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011728512/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828916.981171:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828916.981175:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828916.981178:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011728512:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828916.981184:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011728640:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828916.981204:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828916.981215:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828916/real 1787828916] req@ffff907da562da40 x1874670011728640/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828916.981226:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011728640/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828916.981234:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828916.981237:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828916.981240:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011728640:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828917.108753:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828917.108755:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828918.132696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828918.132784:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828919.156686:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828919.156758:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828920.180688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828920.180730:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828921.204699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828921.204735:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828922.100713:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828922.100723:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828922.100728:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828922.100739:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4325 00000100:00080000:3.0:1787828922.100752:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828922.100759:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828922.100805:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011728768/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828922.100835:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828922.100844:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828922.100846:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828922.100851:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4325 00000100:00080000:3.0:1787828922.100855:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828922.100860:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828922.100872:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828922.100892:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011728896/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828922.100896:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011728768:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787828922.100915:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828922.100920:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828922.100922:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828922.100928:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4325 00000100:00080000:3.0:1787828922.100933:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828922.100938:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828922.100945:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828922.100958:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011729024/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828922.100971:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787828922.100971:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828922.100978:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828922.100981:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828922.100988:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4325 00000100:00080000:3.0:1787828922.101001:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828922.101006:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828922.101012:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828922.101021:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828922/real 1787828922] req@ffff907da562de00 x1874670011728768/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828922.101030:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011729152/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828922.101041:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011728768/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828922.101051:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828922.101055:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828922.101059:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011728768:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828922.101095:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011728896:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828922.101123:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828922.101132:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828922/real 1787828922] req@ffff907da562da40 x1874670011728896/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828922.101140:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011728896/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828922.101146:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828922.101148:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828922.101151:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011728896:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828922.101155:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011729024:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828922.101166:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828922.101172:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828922/real 1787828922] req@ffff907da562cb40 x1874670011729024/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828922.101179:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011729024/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828922.101185:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828922.101187:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828922.101189:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011729024:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828922.101192:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011729152:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828922.101201:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828922.101206:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828922/real 1787828922] req@ffff907da562e1c0 x1874670011729152/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828922.101214:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011729152/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828922.101219:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828922.101221:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828922.101223:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011729152:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828922.228678:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787828922.228760:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828923.252726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828923.252835:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828924.276747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828924.276852:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828925.300761:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828925.300867:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828926.324757:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828926.324882:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828927.220764:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828927.220774:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828927.220779:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828927.220791:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4330 00000100:00080000:2.0:1787828927.220804:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828927.220811:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828927.220870:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011729280/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828927.220899:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828927.220902:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828927.220904:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828927.220907:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4330 00000100:00080000:2.0:1787828927.220910:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828927.220912:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828927.220924:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828927.220933:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011729408/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828927.220944:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787828927.220944:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011729280:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828927.220951:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828927.220954:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828927.220958:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4330 00000100:00080000:2.0:1787828927.220960:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828927.220962:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828927.220966:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828927.220975:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011729536/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828927.220981:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828927.220984:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828927.220985:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828927.220989:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4330 00000100:00080000:2.0:1787828927.220991:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828927.220994:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828927.220997:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828927.221005:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011729664/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787828927.221047:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828927.221114:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828927/real 1787828927] req@ffff907d9114a580 x1874670011729280/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828927.221130:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011729280/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828927.221139:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828927.221143:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828927.221146:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011729280:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828927.221169:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011729408:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828927.221187:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828927.221196:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828927/real 1787828927] req@ffff907d9114ad00 x1874670011729408/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828927.221204:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011729408/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828927.221210:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828927.221212:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828927.221214:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011729408:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828927.221220:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011729536:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828927.221233:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828927.221239:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828927/real 1787828927] req@ffff907d91149a40 x1874670011729536/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828927.221247:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011729536/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828927.221252:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828927.221254:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828927.221256:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011729536:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828927.221259:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011729664:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828927.221270:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828927.221276:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828927/real 1787828927] req@ffff907d9114b840 x1874670011729664/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828927.221283:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011729664/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828927.221288:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828927.221290:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828927.221292:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011729664:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828927.348701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828927.348808:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828928.372698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828928.372749:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828929.396732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828929.396869:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828930.420679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828930.420742:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828931.444674:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828931.444728:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828932.340719:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828932.340730:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828932.340734:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828932.340745:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4336 00000100:00080000:3.0:1787828932.340757:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828932.340764:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828932.340802:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011729792/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828932.340832:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828932.340836:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828932.340838:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828932.340843:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4336 00000100:00080000:3.0:1787828932.340847:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828932.340852:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828932.340863:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828932.340870:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011729792:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828932.340881:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011729920/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828932.340895:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828932.340900:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828932.340902:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828932.340906:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4336 00000100:00080000:3.0:1787828932.340919:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787828932.340924:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787828932.340928:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828932.340936:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828932.340951:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011730048/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828932.340960:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828932/real 1787828932] req@ffff907da562f840 x1874670011729792/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828932.340968:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828932.340974:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828932.340977:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828932.340980:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011729792/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828932.340985:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4336 00000100:00080000:2.0:1787828932.341009:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828932.341009:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828932.341017:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00080000:3.0:1787828932.341018:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828932.341021:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011729792:192.168.203.104@tcp:250:kworker.0 00000008:00100000:3.0:1787828932.341026:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828932.341053:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011730176/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828932.341057:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011729920:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828932.341092:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828932.341103:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828932/real 1787828932] req@ffff907da562f0c0 x1874670011729920/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828932.341111:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011729920/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828932.341117:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828932.341119:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828932.341122:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011729920:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828932.341126:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011730048:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828932.341145:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828932.341152:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828932/real 1787828932] req@ffff907da562c3c0 x1874670011730048/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828932.341159:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011730048/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828932.341164:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828932.341166:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828932.341168:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011730048:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828932.341178:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011730176:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828932.341194:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828932.341202:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828932/real 1787828932] req@ffff907da562e940 x1874670011730176/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828932.341210:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011730176/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828932.341215:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828932.341217:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828932.341220:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011730176:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828932.468732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828932.468797:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828933.492714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828933.492770:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828934.516736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828934.516791:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828935.540717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828935.540761:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828936.564738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828936.564802:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828937.460699:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828937.460708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787828937.460713:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828937.460722:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4341 00000100:00080000:3.0:1787828937.460732:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828937.460738:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787828937.460785:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011730304/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828937.460817:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828937.460821:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787828937.460822:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828937.460826:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4341 00000100:00080000:3.0:1787828937.460829:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828937.460831:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828937.460840:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828937.460850:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011730432/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828937.460859:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828937.460862:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787828937.460863:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828937.460866:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4341 00000100:00080000:3.0:1787828937.460868:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828937.460871:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828937.460874:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787828937.460881:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011730560/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787828937.460890:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828937.460893:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787828937.460895:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787828937.460898:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4341 00000100:00080000:3.0:1787828937.460901:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787828937.460905:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787828937.460909:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828937.460920:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011730304:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828937.460922:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011730688/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828937.461009:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828937.461050:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828937/real 1787828937] req@ffff907da562c000 x1874670011730304/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828937.461067:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011730304/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828937.461077:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828937.461081:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828937.461084:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011730304:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828937.461091:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011730432:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828937.461110:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828937.461120:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828937/real 1787828937] req@ffff907da562fc00 x1874670011730432/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828937.461136:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011730432/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828937.461143:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828937.461145:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828937.461148:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011730432:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828937.461153:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011730560:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828937.461167:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828937.461174:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828937/real 1787828937] req@ffff907da562d2c0 x1874670011730560/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828937.461184:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011730560/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828937.461190:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828937.461193:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828937.461195:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011730560:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828937.461220:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011730688:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828937.461233:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828937.461241:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828937/real 1787828937] req@ffff907da562e580 x1874670011730688/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828937.461250:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011730688/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828937.461262:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828937.461264:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828937.461267:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011730688:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828937.588708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828937.588848:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828938.612700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828938.612774:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828939.636736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828939.636868:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828940.660750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828940.660831:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828941.684760:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828941.684852:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828942.580811:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828942.580828:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828942.580835:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828942.580851:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4346 00000100:00080000:2.0:1787828942.580867:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828942.580877:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828942.580955:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011730816/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828942.581006:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828942.581015:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828942.581020:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828942.581032:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4346 00000100:00080000:2.0:1787828942.581040:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828942.581047:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828942.581066:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828942.581077:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011730816:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828942.581092:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011730944/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828942.581112:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828942.581116:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828942.581119:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828942.581124:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4346 00000100:00080000:2.0:1787828942.581129:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828942.581135:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828942.581143:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828942.581156:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011731072/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828942.581168:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828942.581173:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828942.581176:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1787828942.581185:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787828942.581192:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4346 00000100:00080000:2.0:1787828942.581198:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828942.581202:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828942.581207:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828942.581222:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011731200/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828942.581266:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828942/real 1787828942] req@ffff907d9114b480 x1874670011730816/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828942.581287:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011730816/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828942.581298:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828942.581302:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828942.581306:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011730816:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828942.581336:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011730944:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828942.581358:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828942.581368:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828942/real 1787828942] req@ffff907d91149680 x1874670011730944/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828942.581384:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011730944/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828942.581391:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828942.581393:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828942.581396:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011730944:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828942.581401:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011731072:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828942.581416:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828942.581424:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828942/real 1787828942] req@ffff907d9114bc00 x1874670011731072/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828942.581433:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011731072/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828942.581441:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828942.581445:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828942.581449:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011731072:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828942.581458:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011731200:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828942.581483:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828942.581497:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828942/real 1787828942] req@ffff907d911483c0 x1874670011731200/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828942.581512:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011731200/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828942.581525:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828942.581528:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828942.581531:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011731200:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828942.708754:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828942.708835:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828943.732760:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828943.732905:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828944.756733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828944.756814:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828945.780713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828945.780722:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828946.804763:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828946.804859:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828947.700780:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828947.700789:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828947.700794:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828947.700805:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4351 00000100:00080000:2.0:1787828947.700827:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828947.700836:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828947.700877:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011731328/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828947.700958:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828947.700962:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828947.700965:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828947.700974:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4351 00000100:00080000:2.0:1787828947.700982:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828947.700989:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828947.701009:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828947.701018:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011731328:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828947.701035:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011731456/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828947.701054:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828947.701060:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828947.701063:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828947.701068:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4351 00000100:00080000:2.0:1787828947.701083:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828947.701094:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:3.0:1787828947.701095:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:2.0:1787828947.701106:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828947.701128:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011731584/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828947.701164:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828947.701169:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828947.701171:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828947.701177:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4351 00000100:00080000:2.0:1787828947.701184:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787828947.701190:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828947/real 1787828947] req@ffff907d911483c0 x1874670011731328/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828947.701192:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828947.701201:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828947.701214:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011731712/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787828947.701345:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011731328/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828947.701361:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828947.701367:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828947.701373:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011731328:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828947.701411:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011731456:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787828947.701688:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828947.701859:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828947/real 1787828947] req@ffff907d9114bc00 x1874670011731456/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828947.701941:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011731456/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828947.701959:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828947.701966:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828947.701973:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011731456:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787828947.702015:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011731584:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828947.702065:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828947.702091:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828947/real 1787828947] req@ffff907d91149680 x1874670011731584/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828947.702116:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011731584/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828947.702134:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828947.702141:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828947.702149:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011731584:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828947.702177:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011731712:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787828947.702206:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787828947.702219:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828947/real 1787828947] req@ffff907d9114b480 x1874670011731712/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828947.702251:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011731712/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828947.702263:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828947.702268:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787828947.702273:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011731712:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828947.828771:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828947.828850:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828948.852736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828948.852873:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828949.876740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828949.876834:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828950.900709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828950.900749:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828951.924735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787828951.924784:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828952.820728:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828952.820736:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828952.820740:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828952.820748:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4356 00000100:00080000:2.0:1787828952.820760:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828952.820767:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828952.820801:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011731840/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828952.820826:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828952.820829:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828952.820830:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828952.820833:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4356 00000100:00080000:2.0:1787828952.820836:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828952.820838:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828952.820861:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828952.820876:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011731968/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828952.820883:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011731840:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828952.820888:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828952.820891:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828952.820893:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828952.820896:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4356 00000100:00080000:2.0:1787828952.820899:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828952.820902:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828952.820907:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828952.820923:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011732096/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828952.820933:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828952.820937:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828952.820939:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787828952.820947:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787828952.820949:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4356 00000100:00080000:2.0:1787828952.820956:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828952.820961:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828952.820970:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828952.820986:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011732224/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828952.821002:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828952/real 1787828952] req@ffff907d9114b840 x1874670011731840/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828952.821021:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011731840/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828952.821028:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828952.821032:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828952.821034:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011731840:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828952.821061:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011731968:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828952.821080:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828952.821090:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828952/real 1787828952] req@ffff907d91149a40 x1874670011731968/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828952.821098:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011731968/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828952.821104:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828952.821107:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828952.821109:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011731968:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828952.821114:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011732096:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828952.821149:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828952.821158:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828952/real 1787828952] req@ffff907d9114ad00 x1874670011732096/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828952.821166:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011732096/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828952.821177:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828952.821180:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828952.821182:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011732096:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828952.821211:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011732224:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828952.821227:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828952.821235:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828952/real 1787828952] req@ffff907d9114a580 x1874670011732224/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828952.821244:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011732224/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828952.821251:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828952.821254:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828952.821272:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011732224:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828952.948728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828952.948770:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828953.972730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828953.972779:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828954.996734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828954.996799:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828956.020709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828956.020802:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828957.044687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828957.044831:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828957.940755:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828957.940767:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828957.940772:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828957.940784:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4361 00000100:00080000:1.0:1787828957.940797:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828957.940804:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828957.940876:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011732352/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828957.940909:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828957.940913:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828957.940915:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828957.940920:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4361 00000100:00080000:1.0:1787828957.940923:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828957.940927:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828957.940938:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828957.940951:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011732480/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828957.940964:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828957.940970:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011732352:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828957.940970:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828957.940973:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828957.940983:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4361 00000100:00080000:1.0:1787828957.940987:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828957.940992:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828957.940998:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828957.941009:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011732608/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828957.941016:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828957.941019:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828957.941021:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828957.941024:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4361 00000100:00080000:1.0:1787828957.941036:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828957.941039:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828957.941042:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787828957.941049:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787828957.941053:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011732736/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828957.941106:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828957/real 1787828957] req@ffff907d908a2580 x1874670011732352/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828957.941124:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011732352/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828957.941133:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828957.941137:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828957.941142:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011732352:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828957.941166:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011732480:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828957.941183:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828957.941192:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828957/real 1787828957] req@ffff907d908a21c0 x1874670011732480/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828957.941200:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011732480/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828957.941206:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828957.941208:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828957.941211:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011732480:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828957.941214:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011732608:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828957.941225:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828957.941231:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828957/real 1787828957] req@ffff907d908a3840 x1874670011732608/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828957.941238:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011732608/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828957.941244:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828957.941245:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828957.941247:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011732608:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828957.941250:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011732736:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828957.941260:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828957.941266:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828957/real 1787828957] req@ffff907d908a1680 x1874670011732736/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828957.941274:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011732736/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828957.941279:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828957.941281:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828957.941282:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011732736:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828958.068730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828958.068794:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828959.092734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828959.092840:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828960.116722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828960.116816:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828961.140721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828961.140806:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828962.164687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828962.164771:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828963.060690:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828963.060701:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828963.060706:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828963.060718:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4366 00000100:00080000:1.0:1787828963.060731:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828963.060738:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828963.060797:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011732864/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828963.060825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828963.060828:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828963.060830:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828963.060833:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4366 00000100:00080000:1.0:1787828963.060835:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828963.060838:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828963.060849:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828963.060860:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011732992/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828963.060868:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828963.060871:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828963.060872:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828963.060875:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4366 00000100:00080000:1.0:1787828963.060878:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828963.060883:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011732864:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828963.060884:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828963.060891:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828963.060901:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011733120/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828963.060908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828963.060911:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828963.060912:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828963.060914:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4366 00000100:00080000:1.0:1787828963.060916:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828963.060919:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828963.060921:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828963.060930:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011733248/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828963.060965:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828963.061027:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828963/real 1787828963] req@ffff907d908a0f00 x1874670011732864/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828963.061053:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011732864/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828963.061063:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828963.061067:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828963.061073:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011732864:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828963.061099:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011732992:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828963.061120:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828963.061129:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828963/real 1787828963] req@ffff907d908a12c0 x1874670011732992/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828963.061137:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011732992/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828963.061148:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828963.061150:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828963.061153:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011732992:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828963.061156:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011733120:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828963.061168:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828963.061178:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828963/real 1787828963] req@ffff907d908a1680 x1874670011733120/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828963.061190:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011733120/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828963.061199:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828963.061202:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828963.061206:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011733120:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828963.061213:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011733248:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828963.061232:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828963.061241:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828963/real 1787828963] req@ffff907d908a3840 x1874670011733248/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828963.061249:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011733248/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828963.061255:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828963.061257:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828963.061260:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011733248:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828963.188711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828963.188805:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828964.212724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828964.212789:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828965.236711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828965.236797:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828966.260680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828966.260725:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828967.284703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828967.284768:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828968.180696:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828968.180707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828968.180712:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828968.180724:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4371 00000100:00080000:1.0:1787828968.180735:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828968.180756:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828968.180797:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011733376/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828968.180823:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828968.180826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828968.180827:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828968.180830:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4371 00000100:00080000:1.0:1787828968.180833:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828968.180839:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828968.180849:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828968.180862:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011733504/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828968.180871:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828968.180873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828968.180875:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828968.180879:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4371 00000100:00100000:0.0:1787828968.180886:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011733376:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828968.180886:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828968.180890:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828968.180903:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828968.180913:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011733632/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828968.180921:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828968.180924:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828968.180925:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828968.180928:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4371 00000100:00080000:1.0:1787828968.180931:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828968.180934:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828968.180937:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828968.180946:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011733760/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828968.180966:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828968.181006:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828968/real 1787828968] req@ffff907d908a21c0 x1874670011733376/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828968.181025:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011733376/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828968.181036:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828968.181041:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828968.181046:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011733376:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828968.181070:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011733504:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828968.181095:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828968.181109:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828968/real 1787828968] req@ffff907d908a2580 x1874670011733504/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828968.181122:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011733504/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828968.181131:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828968.181135:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828968.181139:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011733504:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828968.181146:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011733632:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828968.181168:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828968.181181:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828968/real 1787828968] req@ffff907d908a2940 x1874670011733632/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828968.181194:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011733632/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828968.181203:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828968.181207:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828968.181210:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011733632:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828968.181217:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011733760:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828968.181236:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828968.181247:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828968/real 1787828968] req@ffff907d908a0000 x1874670011733760/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828968.181260:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011733760/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828968.181269:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828968.181273:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828968.181276:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011733760:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828968.308702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828968.308827:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828969.332711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828969.332781:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828970.356698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828970.356791:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828971.380766:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828971.380864:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828972.404685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828972.404736:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828973.300675:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828973.300683:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828973.300686:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828973.300694:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4377 00000100:00080000:1.0:1787828973.300703:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828973.300723:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828973.300769:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011733888/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828973.300796:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828973.300800:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828973.300801:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828973.300804:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4377 00000100:00080000:1.0:1787828973.300808:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828973.300813:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828973.300824:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828973.300832:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011733888:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828973.300842:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011734016/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828973.300855:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828973.300859:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828973.300861:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828973.300865:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4377 00000100:00080000:1.0:1787828973.300878:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828973.300883:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:0.0:1787828973.300888:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:1.0:1787828973.300889:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828973.300906:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011734144/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828973.300915:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828973.300918:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828973.300919:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828973.300922:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4377 00000100:00080000:1.0:1787828973.300925:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828973.300928:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828973.300933:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828973.300938:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828973/real 1787828973] req@ffff907d908a2d00 x1874670011733888/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828973.300944:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011734272/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828973.300976:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011733888/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828973.300985:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828973.300988:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828973.300991:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011733888:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828973.301010:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011734016:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828973.301029:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828973.301038:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828973/real 1787828973] req@ffff907d908a1e00 x1874670011734016/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828973.301046:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011734016/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828973.301051:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828973.301053:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828973.301055:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011734016:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828973.301059:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011734144:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828973.301070:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828973.301076:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828973/real 1787828973] req@ffff907d908a0000 x1874670011734144/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828973.301083:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011734144/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828973.301088:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828973.301091:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828973.301092:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011734144:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828973.301097:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011734272:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828973.301107:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828973.301112:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828973/real 1787828973] req@ffff907d908a2940 x1874670011734272/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828973.301119:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011734272/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828973.301125:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828973.301126:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828973.301128:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011734272:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828973.428767:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828973.428851:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828974.452687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828974.452800:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828975.476743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828975.476803:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828976.500671:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828976.500794:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828977.524727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828977.524796:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828978.420663:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828978.420684:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828978.420689:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828978.420700:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4382 00000100:00080000:1.0:1787828978.420712:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828978.420720:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828978.420763:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011734400/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828978.420789:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828978.420792:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828978.420793:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828978.420796:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4382 00000100:00080000:1.0:1787828978.420799:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828978.420802:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828978.420814:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828978.420828:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011734528/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828978.420840:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828978.420844:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828978.420847:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828978.420852:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011734400:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787828978.420854:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4382 00000100:00080000:1.0:1787828978.420863:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828978.420869:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828978.420875:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828978.420895:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011734656/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828978.420904:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828978.420907:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828978.420908:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828978.420911:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4382 00000100:00080000:1.0:1787828978.420922:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828978.420925:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828978.420930:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828978.420943:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011734784/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828978.420990:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011734528:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828978.421031:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011734656:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828978.421062:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011734784:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828978.548722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828978.548816:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828979.572751:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828979.572860:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828980.596694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828980.596765:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787828981.492922:0:2338:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:3.0:1787828981.492991:0:2338:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787828981.493012:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787828981.493062:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787828981.493072:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787828981.493081:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:0.0:1787828981.493126:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828978/real 1787828981] req@ffff907d908a21c0 x1874670011734528/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828981.493175:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011734528/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828981.493191:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828981.493197:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828981.493203:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011734528:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828981.493218:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828978/real 1787828981] req@ffff907d908a3840 x1874670011734656/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828981.493233:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011734656/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828981.493244:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828981.493248:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828981.493251:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011734656:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828981.493259:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828978/real 1787828981] req@ffff907d908a1680 x1874670011734784/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828981.493272:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011734784/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828981.493281:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828981.493284:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828981.493288:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011734784:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828981.493295:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828978/real 1787828981] req@ffff907d908a2580 x1874670011734400/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828981.493308:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011734400/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828981.493316:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828981.493319:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828981.493322:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011734400:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787828981.620717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828981.620799:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828982.644697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828982.644747:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828983.540714:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828983.540723:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787828983.540728:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828983.540739:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4387 00000100:00080000:1.0:1787828983.540750:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828983.540757:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787828983.540790:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011734912/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828983.540815:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828983.540820:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787828983.540823:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828983.540828:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4387 00000100:00080000:1.0:1787828983.540833:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828983.540838:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828983.540848:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828983.540856:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011734912:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787828983.540864:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011735040/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828983.540874:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828983.540877:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828983.540879:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828983.540883:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4387 00000100:00080000:1.0:1787828983.540887:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828983.540891:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828983.540895:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828983.540910:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011735168/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828983.540920:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828983.540924:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787828983.540926:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828983.540930:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4387 00000800:00000100:0.0:1787828983.540936:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787828983.540938:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787828983.540945:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787828983.540951:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787828983.540965:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011735296/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828983.540989:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828983/real 1787828983] req@ffff907d908a12c0 x1874670011734912/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828983.541012:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011734912/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828983.541025:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828983.541029:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828983.541033:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011734912:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828983.541066:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011735040:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828983.541094:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828983.541108:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828983/real 1787828983] req@ffff907d908a0f00 x1874670011735040/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828983.541122:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011735040/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828983.541132:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828983.541136:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828983.541139:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011735040:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828983.541146:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011735168:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828983.541169:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828983.541181:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828983/real 1787828983] req@ffff907d908a2580 x1874670011735168/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828983.541195:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011735168/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828983.541204:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828983.541207:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828983.541211:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011735168:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828983.541217:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011735296:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828983.541241:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828983.541252:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828983/real 1787828983] req@ffff907d908a1680 x1874670011735296/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828983.541264:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011735296/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828983.541273:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828983.541276:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828983.541279:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011735296:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828983.668721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828983.668794:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828984.692685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828984.692758:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828985.716693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828985.716754:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828986.740706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828986.740774:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828987.764706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828987.764760:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828988.660693:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828988.660701:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787828988.660705:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828988.660713:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4392 00000100:00080000:2.0:1787828988.660722:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828988.660727:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787828988.660756:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011735424/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828988.660778:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828988.660784:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787828988.660785:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828988.660788:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4392 00000100:00080000:2.0:1787828988.660791:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828988.660793:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828988.660801:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828988.660810:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011735552/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828988.660819:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828988.660820:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011735424:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787828988.660824:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787828988.660828:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787828988.660832:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4392 00000100:00080000:2.0:1787828988.660836:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828988.660840:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828988.660846:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828988.660861:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011735680/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828988.660872:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828988.660876:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787828988.660878:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787828988.660883:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787828988.660886:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4392 00000100:00080000:2.0:1787828988.660894:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787828988.660899:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787828988.660905:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828988.660916:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011735808/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828988.660922:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828988/real 1787828988] req@ffff907d911492c0 x1874670011735424/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828988.660938:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011735424/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828988.660951:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828988.660954:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828988.660957:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011735424:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828988.660975:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011735552:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787828988.660991:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828988.660999:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828988/real 1787828988] req@ffff907d91148f00 x1874670011735552/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828988.661007:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011735552/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828988.661012:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828988.661014:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828988.661017:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011735552:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787828988.661020:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011735680:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828988.661031:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828988.661037:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828988/real 1787828988] req@ffff907d9114a940 x1874670011735680/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828988.661045:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011735680/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828988.661054:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828988.661057:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828988.661060:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011735680:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787828988.661067:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011735808:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787828988.661087:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828988.661098:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828988/real 1787828988] req@ffff907d91148b40 x1874670011735808/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828988.661108:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011735808/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828988.661113:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828988.661116:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787828988.661118:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011735808:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828988.788711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828988.788769:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828989.812720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828989.812772:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828990.836759:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828990.836785:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828991.860696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828991.860783:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828992.884742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828992.884818:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828993.780726:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828993.780736:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828993.780740:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828993.780749:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4397 00000100:00080000:0.0:1787828993.780760:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828993.780767:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828993.780804:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011735936/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828993.780838:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828993.780842:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828993.780845:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828993.780850:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4397 00000100:00080000:0.0:1787828993.780854:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828993.780860:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828993.780870:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828993.780874:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011735936:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787828993.780887:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011736064/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828993.780900:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828993.780905:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828993.780907:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828993.780912:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4397 00000100:00080000:0.0:1787828993.780916:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828993.780921:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828993.780926:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787828993.780940:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787828993.780941:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011736192/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828993.780954:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828993.780959:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828993.780962:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828993.780966:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4397 00000100:00080000:0.0:1787828993.780970:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828993.780974:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828993.780980:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828993.780989:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828993/real 1787828993] req@ffff907d908470c0 x1874670011735936/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828993.780996:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011736320/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828993.781025:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011735936/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828993.781038:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828993.781044:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828993.781049:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011735936:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828993.781073:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011736064:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828993.781098:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828993.781111:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828993/real 1787828993] req@ffff907d90846580 x1874670011736064/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828993.781120:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011736064/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828993.781125:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828993.781128:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828993.781138:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011736064:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828993.781143:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011736192:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828993.781157:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828993.781164:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828993/real 1787828993] req@ffff907d90847c00 x1874670011736192/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828993.781176:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011736192/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828993.781182:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828993.781184:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828993.781186:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011736192:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828993.781189:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011736320:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828993.781200:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828993.781206:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828993/real 1787828993] req@ffff907d90844b40 x1874670011736320/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828993.781213:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011736320/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828993.781218:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828993.781220:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828993.781222:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011736320:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828993.908705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828993.908763:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828994.932704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787828994.932744:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828995.956729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828995.956824:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828996.980742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828996.980827:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787828998.004730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828998.004783:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828998.900735:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828998.900745:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787828998.900750:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828998.900762:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4402 00000100:00080000:0.0:1787828998.900775:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828998.900783:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787828998.900826:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011736448/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828998.900852:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828998.900855:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787828998.900857:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828998.900859:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4402 00000100:00080000:0.0:1787828998.900862:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828998.900864:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828998.900876:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828998.900891:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011736576/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828998.900905:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787828998.900910:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011736448:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787828998.900911:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787828998.900918:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828998.900925:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4402 00000100:00080000:0.0:1787828998.900932:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828998.900939:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828998.900946:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828998.900963:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011736704/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828998.900975:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828998.900978:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787828998.900980:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828998.900985:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4402 00000800:00000100:2.0:1787828998.900994:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787828998.900997:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787828998.901004:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787828998.901010:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828998.901023:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011736832/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828998.901038:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828998/real 1787828998] req@ffff907d908452c0 x1874670011736448/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828998.901052:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011736448/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828998.901060:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828998.901063:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828998.901066:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011736448:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787828998.901086:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011736576:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787828998.901104:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828998.901113:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828998/real 1787828998] req@ffff907d90844000 x1874670011736576/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828998.901120:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011736576/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828998.901126:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828998.901128:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828998.901131:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011736576:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787828998.901138:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011736704:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828998.901160:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828998.901172:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828998/real 1787828998] req@ffff907d90845e00 x1874670011736704/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828998.901184:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011736704/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828998.901193:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828998.901197:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828998.901201:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011736704:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787828998.901207:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011736832:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787828998.901222:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787828998.901229:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828998/real 1787828998] req@ffff907d908443c0 x1874670011736832/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828998.901249:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011736832/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828998.901255:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828998.901257:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787828998.901259:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011736832:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787828999.028738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787828999.028815:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829000.052684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829000.052763:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829001.076755:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829001.076804:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829002.100680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829002.100750:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829003.124721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829003.124798:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829004.020732:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829004.020743:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829004.020748:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829004.020760:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4407 00000100:00080000:2.0:1787829004.020773:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829004.020782:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829004.020826:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011736960/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829004.020853:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829004.020856:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829004.020858:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829004.020861:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4407 00000100:00080000:2.0:1787829004.020864:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829004.020868:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829004.020880:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829004.020894:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011737088/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829004.020901:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011736960:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829004.020906:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829004.020914:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829004.020918:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829004.020922:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4407 00000100:00080000:2.0:1787829004.020925:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829004.020927:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829004.020930:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829004.020937:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011737216/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829004.020944:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829004.020947:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829004.020948:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829004.020951:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4407 00000100:00080000:2.0:1787829004.020955:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829004.020957:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829004.020962:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829004.020970:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011737344/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829004.020991:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829004.021026:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829004/real 1787829004] req@ffff907d91148b40 x1874670011736960/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829004.021040:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011736960/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829004.021048:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829004.021052:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829004.021055:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011736960:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829004.021074:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011737088:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829004.021090:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829004.021099:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829004/real 1787829004] req@ffff907d9114a940 x1874670011737088/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829004.021108:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011737088/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829004.021114:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829004.021116:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829004.021119:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011737088:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829004.021123:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011737216:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829004.021136:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829004.021142:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829004/real 1787829004] req@ffff907d91148f00 x1874670011737216/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829004.021150:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011737216/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829004.021155:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829004.021158:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829004.021159:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011737216:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829004.021163:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011737344:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829004.021174:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829004.021179:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829004/real 1787829004] req@ffff907d911492c0 x1874670011737344/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829004.021187:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011737344/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829004.021197:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829004.021199:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829004.021201:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011737344:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829004.148674:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829004.148762:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829005.172685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829005.172752:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829006.196760:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829006.196820:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829007.220695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829007.220737:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829008.244723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829008.244761:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829009.140708:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829009.140716:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829009.140726:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829009.140737:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4412 00000100:00080000:2.0:1787829009.140747:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829009.140755:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829009.140792:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011737472/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829009.140815:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829009.140818:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829009.140819:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829009.140822:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4412 00000100:00080000:2.0:1787829009.140825:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829009.140827:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829009.140835:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829009.140849:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011737600/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829009.140859:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829009.140863:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829009.140865:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829009.140870:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4412 00000100:00080000:2.0:1787829009.140874:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829009.140881:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829009.140888:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011737472:192.168.203.104@tcp:250:kworker.0 00000008:00100000:2.0:1787829009.140891:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829009.140915:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011737728/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829009.140930:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829009.140934:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829009.140937:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829009.140941:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4412 00000100:00080000:2.0:1787829009.140946:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829009.140952:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829009.140957:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829009.140971:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011737856/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829009.140985:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829009.141037:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829009/real 1787829009] req@ffff907d9114a580 x1874670011737472/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829009.141058:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011737472/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829009.141072:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829009.141076:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829009.141081:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011737472:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829009.141090:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011737600:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829009.141113:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829009.141125:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829009/real 1787829009] req@ffff907d9114ad00 x1874670011737600/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829009.141138:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011737600/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829009.141149:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829009.141152:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829009.141156:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011737600:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829009.141183:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011737728:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829009.141202:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829009.141213:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829009/real 1787829009] req@ffff907d91149a40 x1874670011737728/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829009.141226:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011737728/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829009.141234:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829009.141238:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829009.141242:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011737728:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829009.141249:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011737856:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829009.141268:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829009.141278:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829009/real 1787829009] req@ffff907d9114b840 x1874670011737856/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829009.141290:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011737856/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829009.141299:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829009.141303:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829009.141307:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011737856:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829009.268706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829009.268764:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829010.292710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829010.292785:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829011.316698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829011.316802:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829012.340693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829012.340749:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829013.364698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829013.364761:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829014.260727:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829014.260739:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829014.260744:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829014.260756:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4418 00000100:00080000:0.0:1787829014.260769:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829014.260777:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829014.260810:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011737984/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829014.260835:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829014.260838:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829014.260840:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829014.260844:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4418 00000100:00080000:0.0:1787829014.260846:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829014.260849:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829014.260858:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829014.260868:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011738112/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829014.260878:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829014.260882:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1787829014.260885:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011737984:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829014.260886:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829014.260896:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4418 00000100:00080000:0.0:1787829014.260899:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829014.260903:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829014.260907:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829014.260918:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011738240/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829014.260931:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829014.260934:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829014.260936:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829014.260938:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4418 00000100:00080000:0.0:1787829014.260941:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829014.260944:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829014.260948:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787829014.260956:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829014.260960:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011738368/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829014.261000:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829014/real 1787829014] req@ffff907d90844780 x1874670011737984/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829014.261015:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011737984/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829014.261024:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829014.261027:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829014.261032:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011737984:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829014.261054:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011738112:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829014.261082:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829014.261098:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829014/real 1787829014] req@ffff907d90845680 x1874670011738112/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829014.261116:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011738112/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829014.261125:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829014.261128:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829014.261131:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011738112:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829014.261136:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011738240:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829014.261156:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829014.261165:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829014/real 1787829014] req@ffff907d90846d00 x1874670011738240/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829014.261175:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011738240/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829014.261181:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829014.261184:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829014.261187:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011738240:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829014.261191:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011738368:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829014.261203:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829014.261212:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829014/real 1787829014] req@ffff907d90845a40 x1874670011738368/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829014.261227:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011738368/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829014.261238:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829014.261242:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829014.261247:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011738368:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829014.388709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829014.388779:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829015.412737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829015.412804:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829016.436665:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829016.436733:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829017.460729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829017.460770:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829018.484703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829018.484755:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829019.380700:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829019.380710:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829019.380714:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829019.380724:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4423 00000100:00080000:0.0:1787829019.380735:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829019.380745:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829019.380802:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011738496/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829019.380853:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829019.380857:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829019.380858:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829019.380862:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4423 00000100:00080000:0.0:1787829019.380866:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829019.380870:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829019.380880:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829019.380895:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011738624/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829019.380907:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829019.380913:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787829019.380914:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011738496:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829019.380916:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829019.380924:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4423 00000100:00080000:0.0:1787829019.380929:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829019.380949:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829019.380955:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829019.380968:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011738752/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829019.380977:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829019.380981:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829019.380982:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829019.380986:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4423 00000100:00080000:0.0:1787829019.381013:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829019.381018:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829019.381023:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787829019.381027:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829019.381041:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011738880/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829019.381102:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829019/real 1787829019] req@ffff907d90844f00 x1874670011738496/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829019.381126:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011738496/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829019.381152:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829019.381158:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829019.381165:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011738496:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829019.381209:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011738624:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829019.381245:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829019.381259:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829019/real 1787829019] req@ffff907d90846940 x1874670011738624/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829019.381286:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011738624/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829019.381294:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829019.381297:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829019.381300:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011738624:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829019.381306:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011738752:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829019.381330:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829019.381359:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829019/real 1787829019] req@ffff907d90847480 x1874670011738752/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829019.381376:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011738752/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829019.381410:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829019.381430:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829019.381434:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011738752:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829019.381454:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011738880:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829019.381479:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829019.381513:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829019/real 1787829019] req@ffff907d90845a40 x1874670011738880/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829019.381529:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011738880/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829019.381541:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829019.381545:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829019.381551:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011738880:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829019.508701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829019.508755:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829020.532742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829020.532801:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829021.556683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829021.556801:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829022.580742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829022.580828:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829023.604683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829023.604722:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829024.500816:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829024.500831:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829024.500837:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829024.500852:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4428 00000100:00080000:2.0:1787829024.500868:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829024.500878:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829024.500925:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011739008/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829024.500954:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829024.500957:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829024.500959:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829024.500972:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4428 00000100:00080000:2.0:1787829024.500977:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829024.500980:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829024.500992:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829024.501005:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011739136/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829024.501014:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829024.501017:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829024.501019:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829024.501024:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4428 00000100:00080000:2.0:1787829024.501026:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829024.501029:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829024.501032:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829024.501041:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011739264/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829024.501050:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011739008:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829024.501054:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829024.501072:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829024.501075:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829024.501080:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4428 00000100:00080000:2.0:1787829024.501082:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829024.501085:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829024.501090:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829024.501108:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011739392/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829024.501186:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829024.501251:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829024/real 1787829024] req@ffff907d9114b480 x1874670011739008/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829024.501273:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011739008/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829024.501284:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829024.501288:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829024.501291:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011739008:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829024.501300:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011739136:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829024.501328:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829024.501341:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829024/real 1787829024] req@ffff907d91149680 x1874670011739136/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829024.501375:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011739136/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829024.501384:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829024.501388:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829024.501393:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011739136:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829024.501420:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011739264:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829024.501442:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829024.501453:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829024/real 1787829024] req@ffff907d9114bc00 x1874670011739264/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829024.501463:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011739264/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829024.501471:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829024.501475:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829024.501478:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011739264:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829024.501486:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011739392:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829024.501501:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829024.501509:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829024/real 1787829024] req@ffff907d911483c0 x1874670011739392/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829024.501525:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011739392/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829024.501532:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829024.501534:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829024.501536:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011739392:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829024.628723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829024.628783:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829025.652699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829025.652822:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829026.676705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829026.676883:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829027.700710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829027.700777:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829028.724708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829028.724803:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829029.620792:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829029.620801:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829029.620806:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829029.620817:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4433 00000100:00080000:0.0:1787829029.620828:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829029.620835:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829029.620871:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011739520/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829029.620905:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829029.620910:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829029.620912:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829029.620916:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4433 00000100:00080000:0.0:1787829029.620920:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829029.620925:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829029.620935:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829029.620949:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011739648/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829029.620957:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011739520:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829029.620968:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829029.620974:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829029.620976:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829029.620981:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4433 00000100:00080000:0.0:1787829029.620986:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829029.620991:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829029.620997:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829029.621010:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011739776/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829029.621019:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787829029.621035:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829029.621039:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829029.621041:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829029.621047:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4433 00000100:00080000:0.0:1787829029.621052:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829029.621056:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829029/real 1787829029] req@ffff907d90846d00 x1874670011739520/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829029.621061:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829029.621071:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829029.621078:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011739520/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829029.621087:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011739904/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829029.621089:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829029.621097:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829029.621104:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011739520:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829029.621126:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011739648:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829029.621149:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829029.621163:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829029/real 1787829029] req@ffff907d90845680 x1874670011739648/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829029.621176:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011739648/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829029.621192:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829029.621196:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829029.621199:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011739648:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829029.621206:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011739776:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829029.621228:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829029.621239:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829029/real 1787829029] req@ffff907d90844780 x1874670011739776/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829029.621247:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011739776/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829029.621252:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829029.621254:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829029.621256:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011739776:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829029.621260:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011739904:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829029.621272:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829029.621278:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829029/real 1787829029] req@ffff907d908443c0 x1874670011739904/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829029.621286:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011739904/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829029.621291:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829029.621292:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829029.621295:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011739904:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829029.748717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829029.748741:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829030.772700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829030.772785:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829031.796726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829031.796784:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829032.820701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829032.820857:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829033.844786:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829033.844913:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829034.740727:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829034.740738:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829034.740743:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829034.740754:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4438 00000100:00080000:2.0:1787829034.740766:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829034.740773:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829034.740816:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011740032/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829034.740841:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829034.740844:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829034.740846:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829034.740849:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4438 00000100:00080000:2.0:1787829034.740853:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829034.740858:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829034.740870:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829034.740886:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011740160/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829034.740891:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011740032:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829034.740907:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829034.740911:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829034.740912:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829034.740917:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4438 00000100:00080000:2.0:1787829034.740920:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829034.740924:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829034.740928:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829034.740938:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011740288/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829034.740945:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829034.740948:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829034.740949:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829034.740952:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4438 00000100:00080000:2.0:1787829034.740956:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829034.740958:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829034.740961:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787829034.740968:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829034.740971:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011740416/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829034.741013:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829034/real 1787829034] req@ffff907d911483c0 x1874670011740032/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829034.741029:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011740032/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829034.741039:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829034.741043:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829034.741046:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011740032:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829034.741066:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011740160:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829034.741087:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829034.741101:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829034/real 1787829034] req@ffff907d9114bc00 x1874670011740160/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829034.741115:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011740160/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829034.741126:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829034.741129:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829034.741134:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011740160:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829034.741140:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011740288:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829034.741158:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829034.741167:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829034/real 1787829034] req@ffff907d91149680 x1874670011740288/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829034.741175:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011740288/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829034.741181:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829034.741183:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829034.741186:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011740288:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829034.741190:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011740416:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829034.741200:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829034.741206:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829034/real 1787829034] req@ffff907d9114b480 x1874670011740416/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829034.741213:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011740416/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829034.741219:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829034.741221:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829034.741222:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011740416:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829034.868697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829034.868748:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829035.892704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829035.892759:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829036.916702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829036.916813:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829037.940714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829037.940790:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829038.964688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829038.964761:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829039.860694:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829039.860705:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829039.860710:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829039.860719:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4443 00000100:00080000:2.0:1787829039.860730:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829039.860739:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829039.860774:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011740544/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829039.860801:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829039.860806:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829039.860808:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829039.860812:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4443 00000100:00080000:2.0:1787829039.860817:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829039.860822:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829039.860832:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829039.860843:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011740544:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829039.860851:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011740672/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829039.860866:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829039.860870:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829039.860872:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829039.860876:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4443 00000100:00080000:2.0:1787829039.860880:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829039.860885:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829039.860891:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787829039.860897:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829039.860906:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011740800/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829039.860918:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829039.860922:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829039.860925:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829039.860931:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829039/real 1787829039] req@ffff907d9114b840 x1874670011740544/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829039.860931:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4443 00000100:00080000:2.0:1787829039.860939:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829039.860948:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829039.860961:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011740544/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829039.860964:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1787829039.860975:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829039.860979:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829039.860985:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011740544:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829039.860985:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011740928/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829039.861011:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011740672:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829039.861039:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829039.861054:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829039/real 1787829039] req@ffff907d91149a40 x1874670011740672/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829039.861067:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011740672/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829039.861076:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829039.861080:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829039.861084:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011740672:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829039.861090:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011740800:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829039.861114:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829039.861123:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829039/real 1787829039] req@ffff907d9114ad00 x1874670011740800/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829039.861131:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011740800/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829039.861140:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829039.861142:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829039.861145:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011740800:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829039.861148:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011740928:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829039.861158:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829039.861164:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829039/real 1787829039] req@ffff907d9114a580 x1874670011740928/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829039.861171:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011740928/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829039.861176:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829039.861177:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829039.861179:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011740928:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829039.988694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829039.988773:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829041.012662:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829041.012740:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829042.036690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829042.036751:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829043.060682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829043.060764:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829044.084738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829044.084810:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829044.980679:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829044.980691:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829044.980695:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829044.980706:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4448 00000100:00080000:2.0:1787829044.980718:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829044.980726:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829044.980770:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011741056/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829044.980795:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829044.980798:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829044.980799:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829044.980803:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4448 00000100:00080000:2.0:1787829044.980805:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829044.980807:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829044.980817:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829044.980828:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011741184/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829044.980836:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829044.980839:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829044.980840:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829044.980845:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011741056:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829044.980846:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4448 00000100:00080000:2.0:1787829044.980851:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829044.980856:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829044.980859:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829044.980891:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011741312/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829044.980900:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829044.980903:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829044.980904:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829044.980907:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4448 00000100:00080000:2.0:1787829044.980911:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829044.980914:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829044.980918:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829044.980927:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011741440/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829044.981007:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011741184:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829044.981055:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011741312:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829044.981073:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011741440:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829045.108734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829045.108806:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829046.132697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829046.132761:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829047.156708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829047.156764:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829048.052974:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787829048.053037:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829048.053057:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829048.053103:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829048.053118:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829048.053138:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:2.0:1787829048.053165:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829044/real 1787829048] req@ffff907d91148f00 x1874670011741184/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829048.053217:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011741184/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829048.053232:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829048.053239:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829048.053244:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011741184:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829048.053256:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829044/real 1787829048] req@ffff907d9114a940 x1874670011741312/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829048.053273:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011741312/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829048.053282:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829048.053286:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829048.053290:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011741312:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829048.053299:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829044/real 1787829048] req@ffff907d91148b40 x1874670011741440/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829048.053312:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011741440/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829048.053322:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829048.053326:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829048.053330:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011741440:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829048.053340:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829044/real 1787829048] req@ffff907d911492c0 x1874670011741056/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829048.053357:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011741056/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829048.053368:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829048.053372:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829048.053376:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011741056:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829048.180696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829048.180787:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829049.204684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829049.204730:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829050.100733:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829050.100747:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829050.100753:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829050.100767:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4453 00000100:00080000:1.0:1787829050.100781:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829050.100790:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829050.100842:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011741568/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829050.100871:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829050.100875:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829050.100877:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829050.100881:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4453 00000100:00080000:1.0:1787829050.100886:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829050.100891:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829050.100904:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829050.100920:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011741568:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829050.100922:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011741696/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829050.100939:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829050.100942:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829050.100945:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829050.100950:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4453 00000100:00080000:1.0:1787829050.100954:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829050.100958:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829050.100964:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829050.100975:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011741824/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829050.100985:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829050.100988:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829050.100991:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829050.100995:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4453 00000100:00080000:1.0:1787829050.100998:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829050.101003:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829050.101007:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829050.101017:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011741952/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829050.101024:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829050.101085:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829050/real 1787829050] req@ffff907d908a3840 x1874670011741568/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829050.101107:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011741568/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829050.101120:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829050.101125:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829050.101130:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011741568:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829050.101167:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011741696:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829050.101208:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829050.101224:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829050/real 1787829050] req@ffff907d908a21c0 x1874670011741696/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829050.101237:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011741696/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829050.101252:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829050.101255:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829050.101259:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011741696:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829050.101267:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011741824:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829050.101290:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829050.101302:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829050/real 1787829050] req@ffff907d908a2940 x1874670011741824/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829050.101315:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011741824/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829050.101325:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829050.101328:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829050.101332:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011741824:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829050.101339:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011741952:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829050.101368:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829050.101382:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829050/real 1787829050] req@ffff907d908a0000 x1874670011741952/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829050.101395:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011741952/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829050.101410:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829050.101414:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829050.101418:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011741952:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829050.228683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829050.228714:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829051.252717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829051.252781:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829052.276719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829052.276792:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829053.300751:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829053.300838:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829054.324700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829054.324780:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829055.220738:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829055.220751:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829055.220757:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829055.220771:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4458 00000100:00080000:1.0:1787829055.220785:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829055.220799:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829055.220853:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011742080/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829055.220888:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829055.220892:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829055.220896:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829055.220904:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4458 00000100:00080000:1.0:1787829055.220910:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829055.220917:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829055.220932:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829055.220937:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011742080:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829055.220952:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011742208/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829055.220969:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829055.220973:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829055.220977:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829055.220983:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4458 00000100:00080000:1.0:1787829055.220989:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829055.220995:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829055.221002:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787829055.221012:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829055.221020:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011742336/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829055.221034:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829055.221038:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829055.221041:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829055.221048:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4458 00000100:00080000:1.0:1787829055.221054:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829055.221056:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829055/real 1787829055] req@ffff907d908a1e00 x1874670011742080/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829055.221068:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829055.221086:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829055.221093:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011742080/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829055.221104:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787829055.221106:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011742464/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829055.221115:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829055.221152:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011742080:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829055.221185:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011742208:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829055.221210:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829055.221228:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829055/real 1787829055] req@ffff907d908a2d00 x1874670011742208/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829055.221238:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011742208/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829055.221245:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829055.221248:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829055.221251:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011742208:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829055.221256:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011742336:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829055.221272:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829055.221289:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829055/real 1787829055] req@ffff907d908a0000 x1874670011742336/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829055.221303:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011742336/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829055.221317:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829055.221322:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829055.221328:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011742336:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829055.221341:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011742464:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829055.221382:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829055.221412:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829055/real 1787829055] req@ffff907d908a2940 x1874670011742464/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829055.221421:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011742464/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829055.221426:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829055.221428:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829055.221431:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011742464:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829055.348731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829055.348812:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829056.372697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829056.372720:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829057.396707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829057.396751:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829058.420687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829058.420781:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829059.444713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829059.444768:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829060.340746:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829060.340762:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829060.340767:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829060.340779:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4464 00000100:00080000:1.0:1787829060.340793:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829060.340800:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829060.340847:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011742592/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829060.340870:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829060.340873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829060.340874:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829060.340877:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4464 00000100:00080000:1.0:1787829060.340880:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829060.340882:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829060.340891:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829060.340906:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011742720/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829060.340920:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829060.340923:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829060.340925:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829060.340933:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4464 00000100:00080000:1.0:1787829060.340937:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829060.340939:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011742592:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829060.340942:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829060.340955:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829060.340964:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011742848/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829060.340971:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829060.340974:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829060.340975:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829060.340978:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4464 00000100:00080000:1.0:1787829060.340980:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829060.340982:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829060.340986:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829060.340993:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011742976/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829060.341025:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829060.341065:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829060/real 1787829060] req@ffff907d908a21c0 x1874670011742592/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829060.341083:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011742592/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829060.341094:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829060.341114:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829060.341119:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011742592:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829060.341145:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011742720:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829060.341170:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829060.341183:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829060/real 1787829060] req@ffff907d908a3840 x1874670011742720/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829060.341196:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011742720/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829060.341205:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829060.341209:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829060.341212:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011742720:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829060.341219:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011742848:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829060.341237:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829060.341247:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829060/real 1787829060] req@ffff907d908a1680 x1874670011742848/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829060.341266:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011742848/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829060.341275:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829060.341278:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829060.341282:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011742848:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829060.341289:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011742976:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829060.341307:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829060.341318:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829060/real 1787829060] req@ffff907d908a2580 x1874670011742976/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829060.341330:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011742976/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829060.341340:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829060.341343:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829060.341347:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011742976:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829060.468699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829060.468776:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829061.492679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829061.492812:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829062.516681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829062.516804:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829063.540748:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829063.540770:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829064.564696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829064.564776:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829065.460738:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829065.460753:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829065.460759:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829065.460772:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4469 00000100:00080000:2.0:1787829065.460785:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829065.460794:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829065.460836:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011743104/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829065.460865:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829065.460868:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829065.460869:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829065.460873:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4469 00000100:00080000:2.0:1787829065.460876:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829065.460879:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829065.460890:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829065.460900:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011743104:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829065.460902:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011743232/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829065.460915:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829065.460917:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829065.460918:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829065.460921:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4469 00000100:00080000:2.0:1787829065.460923:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829065.460926:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829065.460929:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829065.460936:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011743360/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829065.460942:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829065.460945:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829065.460947:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829065.460951:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4469 00000100:00080000:2.0:1787829065.460954:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829065.460956:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829065.460959:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829065.460966:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011743488/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829065.460996:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829065.461043:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829065/real 1787829065] req@ffff907d911492c0 x1874670011743104/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829065.461094:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011743104/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829065.461103:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829065.461106:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829065.461110:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011743104:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829065.461132:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011743232:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829065.461150:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829065.461160:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829065/real 1787829065] req@ffff907d91148b40 x1874670011743232/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829065.461168:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011743232/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829065.461174:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829065.461176:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829065.461178:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011743232:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829065.461182:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011743360:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829065.461194:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829065.461200:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829065/real 1787829065] req@ffff907d9114a940 x1874670011743360/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829065.461208:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011743360/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829065.461218:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829065.461220:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829065.461222:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011743360:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829065.461226:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011743488:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829065.461237:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829065.461242:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829065/real 1787829065] req@ffff907d91148f00 x1874670011743488/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829065.461249:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011743488/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829065.461255:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829065.461257:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829065.461259:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011743488:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829065.588742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829065.588810:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829066.612714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829066.612782:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829067.636689:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829067.636756:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829068.660763:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829068.660870:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829069.684721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829069.684814:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829070.580787:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829070.580798:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829070.580803:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829070.580813:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4474 00000100:00080000:2.0:1787829070.580826:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829070.580833:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829070.580902:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011743616/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829070.580945:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829070.580948:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829070.580950:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829070.580953:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4474 00000100:00080000:2.0:1787829070.580956:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829070.580960:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829070.580971:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829070.580984:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011743744/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829070.580992:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829070.580994:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829070.580996:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829070.580998:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4474 00000100:00080000:2.0:1787829070.581000:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829070.581003:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829070.581007:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829070.581014:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011743872/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829070.581021:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829070.581024:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829070.581025:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829070.581041:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4474 00000100:00080000:2.0:1787829070.581045:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829070.581047:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829070.581051:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829070.581060:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011744000/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829070.581085:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011743616:192.168.203.104@tcp:250:kworker.0 00000800:00000100:1.0:1787829070.581168:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829070.581216:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829070/real 1787829070] req@ffff907d91148f00 x1874670011743616/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829070.581238:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011743616/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829070.581248:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829070.581252:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829070.581255:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011743616:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829070.581272:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011743744:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829070.581293:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829070.581302:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829070/real 1787829070] req@ffff907d9114a940 x1874670011743744/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829070.581311:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011743744/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829070.581317:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829070.581319:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829070.581322:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011743744:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829070.581326:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011743872:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829070.581337:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829070.581344:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829070/real 1787829070] req@ffff907d91148b40 x1874670011743872/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829070.581363:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011743872/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829070.581369:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829070.581371:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829070.581373:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011743872:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829070.581393:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011744000:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829070.581405:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829070.581411:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829070/real 1787829070] req@ffff907d911492c0 x1874670011744000/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829070.581420:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011744000/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829070.581426:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829070.581428:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829070.581430:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011744000:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787829070.708742:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829070.708767:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787829071.732786:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829071.732930:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829072.756811:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829072.756870:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829073.780762:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829073.780821:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829074.804754:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829074.804837:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829075.700747:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829075.700760:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829075.700781:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829075.700792:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4479 00000100:00080000:2.0:1787829075.700804:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829075.700812:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829075.700868:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011744128/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829075.700899:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829075.700903:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829075.700905:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829075.700909:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4479 00000100:00080000:2.0:1787829075.700913:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829075.700917:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829075.700927:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829075.700940:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011744256/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829075.700967:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829075.700970:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829075.700971:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829075.700974:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4479 00000100:00080000:2.0:1787829075.700977:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829075.700980:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829075.700985:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829075.701013:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011744128:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829075.701013:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011744384/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829075.701040:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829075.701043:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829075.701044:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829075.701050:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4479 00000100:00080000:2.0:1787829075.701053:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829075.701057:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829075.701063:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829075.701073:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011744512/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829075.701114:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829075.701164:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829075/real 1787829075] req@ffff907d9114a580 x1874670011744128/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829075.701179:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011744128/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829075.701189:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829075.701192:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829075.701195:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011744128:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829075.701201:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011744256:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829075.701217:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829075.701225:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829075/real 1787829075] req@ffff907d9114ad00 x1874670011744256/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829075.701242:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011744256/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829075.701249:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829075.701252:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829075.701254:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011744256:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829075.701278:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011744384:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829075.701290:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829075.701297:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829075/real 1787829075] req@ffff907d91149a40 x1874670011744384/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829075.701304:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011744384/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829075.701310:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829075.701312:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829075.701314:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011744384:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829075.701319:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011744512:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829075.701328:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829075.701333:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829075/real 1787829075] req@ffff907d9114b840 x1874670011744512/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829075.701341:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011744512/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829075.701346:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829075.701348:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829075.701351:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011744512:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829075.828741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829075.828823:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829076.852711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829076.852782:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829077.876709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829077.876761:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829078.900701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829078.900882:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829079.924740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829079.924836:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829080.820677:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829080.820687:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829080.820691:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829080.820699:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4484 00000100:00080000:2.0:1787829080.820709:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829080.820715:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829080.820749:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011744640/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829080.820773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829080.820777:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829080.820778:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829080.820781:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4484 00000100:00080000:2.0:1787829080.820784:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829080.820786:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829080.820795:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829080.820812:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011744768/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829080.820822:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829080.820825:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011744640:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829080.820826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829080.820829:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829080.820838:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4484 00000100:00080000:2.0:1787829080.820841:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829080.820844:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829080.820848:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829080.820858:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011744896/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829080.820865:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829080.820868:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829080.820870:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829080.820873:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4484 00000100:00080000:2.0:1787829080.820875:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829080.820878:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829080.820882:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829080.820890:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011745024/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829080.820896:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829080.820942:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829080/real 1787829080] req@ffff907d9114b480 x1874670011744640/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829080.820982:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011744640/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829080.820995:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829080.821001:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829080.821023:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011744640:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829080.821058:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011744768:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829080.821098:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829080.821134:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829080/real 1787829080] req@ffff907d91149680 x1874670011744768/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829080.821152:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011744768/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829080.821163:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829080.821168:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829080.821173:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011744768:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829080.821181:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011744896:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829080.821208:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829080.821246:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829080/real 1787829080] req@ffff907d9114bc00 x1874670011744896/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829080.821258:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011744896/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829080.821266:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829080.821270:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829080.821273:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011744896:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829080.821278:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011745024:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829080.821296:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829080.821305:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829080/real 1787829080] req@ffff907d911483c0 x1874670011745024/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829080.821316:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011745024/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829080.821323:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829080.821325:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829080.821327:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011745024:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829080.948722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829080.948827:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829081.972693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829081.972917:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829082.996702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829082.996797:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829084.020739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829084.020826:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829085.044715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829085.044797:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829085.940753:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829085.940765:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829085.940770:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829085.940781:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4489 00000100:00080000:0.0:1787829085.940793:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829085.940800:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829085.940843:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011745152/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829085.940865:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829085.940868:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829085.940870:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829085.940875:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4489 00000100:00080000:0.0:1787829085.940877:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829085.940882:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829085.940892:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829085.940903:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011745280/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829085.940911:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829085.940914:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829085.940917:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829085.940917:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011745152:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829085.940935:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4489 00000100:00080000:0.0:1787829085.940939:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829085.940943:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829085.940947:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829085.940958:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011745408/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829085.940965:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829085.940968:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829085.940969:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829085.940972:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4489 00000100:00080000:0.0:1787829085.940987:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829085.940990:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829085.940994:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829085.941003:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011745536/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829085.941005:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829085.941051:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829085/real 1787829085] req@ffff907d90845e00 x1874670011745152/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829085.941071:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011745152/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829085.941079:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829085.941082:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829085.941085:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011745152:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829085.941107:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011745280:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829085.941123:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829085.941131:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829085/real 1787829085] req@ffff907d90844000 x1874670011745280/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829085.941139:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011745280/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829085.941146:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829085.941148:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829085.941150:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011745280:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829085.941154:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011745408:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829085.941174:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829085.941186:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829085/real 1787829085] req@ffff907d908452c0 x1874670011745408/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829085.941199:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011745408/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829085.941216:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829085.941220:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829085.941224:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011745408:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829085.941231:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011745536:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829085.941252:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829085.941264:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829085/real 1787829085] req@ffff907d90844b40 x1874670011745536/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829085.941281:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011745536/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829085.941288:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829085.941290:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829085.941292:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011745536:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829086.068778:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829086.068849:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829087.092682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829087.092737:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829088.116732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829088.116852:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829089.140683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829089.140743:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829090.164703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829090.164751:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829091.060709:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829091.060718:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829091.060722:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829091.060731:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4494 00000100:00080000:0.0:1787829091.060741:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829091.060747:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829091.060776:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011745664/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829091.060798:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829091.060802:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829091.060804:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829091.060808:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4494 00000100:00080000:0.0:1787829091.060813:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829091.060818:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829091.060828:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829091.060844:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011745792/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829091.060857:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829091.060857:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011745664:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829091.060861:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829091.060863:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829091.060867:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4494 00000100:00080000:0.0:1787829091.060871:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829091.060876:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829091.060882:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829091.060908:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011745920/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829091.060918:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829091.060921:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829091.060925:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829091.060928:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4494 00000100:00080000:0.0:1787829091.060931:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787829091.060932:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787829091.060934:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829091.060940:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829091.060950:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011746048/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829091.060972:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829091/real 1787829091] req@ffff907d90847c00 x1874670011745664/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829091.060991:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011745664/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829091.061001:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829091.061004:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829091.061007:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011745664:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829091.061029:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011745792:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829091.061057:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829091.061075:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829091/real 1787829091] req@ffff907d90846580 x1874670011745792/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829091.061088:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011745792/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829091.061097:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829091.061102:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829091.061105:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011745792:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829091.061112:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011745920:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829091.061130:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829091.061138:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829091/real 1787829091] req@ffff907d908470c0 x1874670011745920/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829091.061145:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011745920/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829091.061150:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829091.061153:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829091.061155:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011745920:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829091.061159:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011746048:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829091.061169:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829091.061175:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829091/real 1787829091] req@ffff907d90844b40 x1874670011746048/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829091.061184:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011746048/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829091.061189:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829091.061191:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829091.061194:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011746048:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829091.188694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829091.188736:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829092.212700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829092.212756:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829093.236694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829093.236758:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829094.260683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829094.260774:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829095.284703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829095.284782:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829096.180743:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829096.180754:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829096.180758:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829096.180769:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4499 00000100:00080000:2.0:1787829096.180781:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829096.180789:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829096.180832:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011746176/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829096.180872:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829096.180877:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829096.180879:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829096.180884:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4499 00000100:00080000:2.0:1787829096.180889:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829096.180894:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829096.180905:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829096.180921:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011746304/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829096.180934:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011746176:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829096.180939:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829096.180950:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829096.180952:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829096.180958:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4499 00000100:00080000:2.0:1787829096.180962:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829096.180967:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829096.180973:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829096.180989:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011746432/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829096.181000:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829096.181004:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829096.181006:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829096.181010:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4499 00000100:00080000:2.0:1787829096.181024:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829096.181029:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:1.0:1787829096.181038:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:2.0:1787829096.181042:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829096.181059:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011746560/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829096.181111:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829096/real 1787829096] req@ffff907d911483c0 x1874670011746176/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829096.181142:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011746176/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829096.181155:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829096.181161:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829096.181165:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011746176:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829096.181196:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011746304:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829096.181224:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829096.181239:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829096/real 1787829096] req@ffff907d9114bc00 x1874670011746304/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829096.181252:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011746304/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829096.181268:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829096.181272:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829096.181276:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011746304:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829096.181283:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011746432:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829096.181307:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829096.181321:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829096/real 1787829096] req@ffff907d91149680 x1874670011746432/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829096.181334:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011746432/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829096.181343:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829096.181347:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829096.181351:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011746432:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829096.181367:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011746560:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829096.181391:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829096.181404:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829096/real 1787829096] req@ffff907d9114b480 x1874670011746560/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829096.181417:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011746560/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829096.181425:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829096.181428:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829096.181432:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011746560:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829096.308735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829096.308812:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829097.332705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829097.332782:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829098.356707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829098.356794:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829099.380681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829099.380787:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829100.404713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829100.404811:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829101.300739:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829101.300749:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829101.300753:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829101.300763:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4505 00000100:00080000:0.0:1787829101.300775:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829101.300782:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829101.300829:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011746688/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829101.300860:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829101.300865:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829101.300867:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829101.300872:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4505 00000100:00080000:0.0:1787829101.300876:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829101.300880:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829101.300890:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829101.300907:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011746816/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829101.300915:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011746688:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829101.300928:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829101.300932:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829101.300934:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829101.300940:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4505 00000100:00080000:0.0:1787829101.300945:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829101.300949:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829101.300955:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829101.300969:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011746944/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829101.300981:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787829101.300986:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787829101.300988:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829101.300992:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829101.300997:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4505 00000100:00080000:0.0:1787829101.301002:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829101.301007:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829101.301012:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829101.301025:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011747072/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829101.301035:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829101/real 1787829101] req@ffff907d908452c0 x1874670011746688/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829101.301067:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011746688/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829101.301078:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829101.301082:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829101.301087:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011746688:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829101.301108:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011746816:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829101.301132:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829101.301147:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829101/real 1787829101] req@ffff907d90844000 x1874670011746816/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829101.301159:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011746816/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829101.301168:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829101.301171:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829101.301174:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011746816:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829101.301181:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011746944:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829101.301202:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829101.301224:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829101/real 1787829101] req@ffff907d90845e00 x1874670011746944/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829101.301236:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011746944/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829101.301244:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829101.301247:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829101.301250:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011746944:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829101.301256:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011747072:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829101.301275:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829101.301287:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829101/real 1787829101] req@ffff907d908443c0 x1874670011747072/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829101.301298:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011747072/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829101.301306:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829101.301309:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829101.301313:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011747072:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829101.428706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829101.428732:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829102.452719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829102.452804:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829103.476670:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829103.476751:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829104.500682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829104.500747:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829105.524713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829105.524759:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829106.420735:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829106.420745:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829106.420750:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829106.420760:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4510 00000100:00080000:1.0:1787829106.420771:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829106.420794:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829106.420839:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011747200/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829106.420871:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829106.420876:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829106.420878:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829106.420882:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4510 00000100:00080000:1.0:1787829106.420886:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829106.420892:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829106.420902:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829106.420913:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011747200:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829106.420920:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011747328/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829106.420935:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829106.420939:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829106.420941:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829106.420946:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4510 00000100:00080000:1.0:1787829106.420951:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829106.420956:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829106.420962:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787829106.420969:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829106.420979:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011747456/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829106.420992:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829106.420995:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829106.420998:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829106.421005:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4510 00000100:00100000:2.0:1787829106.421006:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829106/real 1787829106] req@ffff907d908a0f00 x1874670011747200/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829106.421012:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829106.421022:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829106.421027:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011747200/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829106.421030:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1787829106.421041:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829106.421044:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829106.421050:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011747200:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829106.421052:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011747584/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829106.421072:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011747328:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829106.421090:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829106.421100:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829106/real 1787829106] req@ffff907d908a12c0 x1874670011747328/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829106.421109:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011747328/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829106.421115:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829106.421118:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829106.421120:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011747328:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829106.421123:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011747456:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829106.421134:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829106.421140:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829106/real 1787829106] req@ffff907d908a2580 x1874670011747456/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829106.421147:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011747456/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829106.421153:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829106.421155:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829106.421157:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011747456:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829106.421160:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011747584:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829106.421169:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829106.421174:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829106/real 1787829106] req@ffff907d908a1680 x1874670011747584/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829106.421181:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011747584/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829106.421186:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829106.421187:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829106.421190:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011747584:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829106.548677:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829106.548742:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829107.572697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829107.572739:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829108.596694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829108.596787:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829109.620727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829109.620797:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829110.644691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829110.644733:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829111.540708:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829111.540720:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829111.540725:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829111.540737:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4515 00000100:00080000:1.0:1787829111.540760:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829111.540767:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829111.540810:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011747712/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829111.540836:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829111.540839:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829111.540841:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829111.540844:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4515 00000100:00080000:1.0:1787829111.540846:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829111.540849:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829111.540859:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829111.540869:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011747840/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829111.540883:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829111.540885:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829111.540887:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829111.540890:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4515 00000100:00080000:1.0:1787829111.540892:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829111.540894:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829111.540897:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829111.540906:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011747968/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829111.540914:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829111.540917:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829111.540921:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829111.540926:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4515 00000100:00100000:2.0:1787829111.540927:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011747712:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829111.540929:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829111.540940:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829111.540948:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829111.540956:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011748096/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829111.541040:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011747840:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829111.541075:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011747968:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829111.541092:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011748096:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829111.668721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829111.668810:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829112.692697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829112.692784:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829113.716685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829113.716757:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829114.612827:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787829114.612867:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829114.612881:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829114.612909:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829114.612916:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829114.612923:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:2.0:1787829114.612963:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829111/real 1787829114] req@ffff907d908a2940 x1874670011747968/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829114.612990:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011747968/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829114.613004:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829114.613009:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829114.613014:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011747968:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829114.613022:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829111/real 1787829114] req@ffff907d908a0000 x1874670011748096/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829114.613035:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011748096/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829114.613044:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829114.613048:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829114.613052:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011748096:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829114.613059:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829111/real 1787829114] req@ffff907d908a3840 x1874670011747712/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829114.613072:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011747712/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829114.613081:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829114.613085:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829114.613088:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011747712:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829114.613096:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829111/real 1787829114] req@ffff907d908a21c0 x1874670011747840/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829114.613107:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011747840/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829114.613115:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829114.613118:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829114.613122:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011747840:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829114.740674:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829114.740755:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829115.764699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829115.764746:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829116.660681:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829116.660689:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829116.660693:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829116.660701:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4520 00000100:00080000:2.0:1787829116.660734:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829116.660740:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829116.660767:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011748224/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829116.660790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829116.660792:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829116.660794:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829116.660797:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4520 00000100:00080000:2.0:1787829116.660800:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829116.660803:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829116.660811:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829116.660822:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011748224:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829116.660823:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011748352/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829116.660838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829116.660842:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829116.660844:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829116.660848:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4520 00000100:00080000:2.0:1787829116.660851:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829116.660855:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829116.660861:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829116.660874:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011748480/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829116.660885:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829116.660889:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829116.660891:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829116.660895:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4520 00000100:00080000:2.0:1787829116.660899:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829116.660907:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:1.0:1787829116.660908:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:2.0:1787829116.660917:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829116.660931:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011748608/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829116.660957:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829116/real 1787829116] req@ffff907d9114b840 x1874670011748224/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829116.660978:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011748224/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829116.660989:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829116.660994:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829116.660998:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011748224:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829116.661021:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011748352:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829116.661048:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829116.661062:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829116/real 1787829116] req@ffff907d91149a40 x1874670011748352/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829116.661075:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011748352/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829116.661085:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829116.661088:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829116.661092:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011748352:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829116.661099:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011748480:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829116.661123:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829116.661144:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829116/real 1787829116] req@ffff907d9114ad00 x1874670011748480/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829116.661156:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011748480/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829116.661165:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829116.661169:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829116.661172:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011748480:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829116.661179:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011748608:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829116.661200:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829116.661212:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829116/real 1787829116] req@ffff907d9114a580 x1874670011748608/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829116.661225:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011748608/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829116.661233:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829116.661237:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829116.661240:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011748608:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829116.788701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829116.788786:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829117.812680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829117.812744:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829118.836691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829118.836753:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829119.860724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829119.860774:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829120.884667:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829120.884745:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829121.780708:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829121.780719:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829121.780725:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829121.780736:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4525 00000100:00080000:0.0:1787829121.780750:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829121.780758:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829121.780808:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011748736/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829121.780845:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829121.780850:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829121.780852:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829121.780857:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4525 00000100:00080000:0.0:1787829121.780862:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829121.780867:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829121.780879:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829121.780894:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011748864/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829121.780908:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011748736:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829121.780909:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829121.780917:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829121.780921:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829121.780926:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4525 00000100:00080000:0.0:1787829121.780929:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829121.780934:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829121.780941:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829121.780953:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011748992/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829121.780965:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829121.780969:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829121.780971:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829121.780975:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4525 00000100:00080000:0.0:1787829121.780979:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829121.780984:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829121.780989:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787829121.780993:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829121.781005:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011749120/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829121.781051:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829121/real 1787829121] req@ffff907d90844780 x1874670011748736/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829121.781076:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011748736/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829121.781088:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829121.781093:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829121.781098:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011748736:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829121.781125:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011748864:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829121.781146:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829121.781156:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829121/real 1787829121] req@ffff907d90845680 x1874670011748864/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829121.781165:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011748864/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829121.781171:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829121.781174:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829121.781176:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011748864:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829121.781180:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011748992:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829121.781192:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829121.781199:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829121/real 1787829121] req@ffff907d90846d00 x1874670011748992/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829121.781207:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011748992/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829121.781216:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829121.781219:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829121.781221:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011748992:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829121.781224:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011749120:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829121.781234:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829121.781239:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829121/real 1787829121] req@ffff907d90845a40 x1874670011749120/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829121.781247:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011749120/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829121.781252:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829121.781254:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829121.781256:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011749120:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829121.908720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829121.908788:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829122.932709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829122.932784:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829123.956712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829123.956792:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829124.980683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829124.980757:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829126.004719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829126.004782:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829126.900708:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829126.900714:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829126.900719:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829126.900730:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4530 00000100:00080000:0.0:1787829126.900742:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829126.900750:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829126.900785:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011749248/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829126.900816:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829126.900820:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829126.900822:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829126.900827:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4530 00000100:00080000:0.0:1787829126.900832:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829126.900836:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829126.900845:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829126.900853:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011749248:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829126.900863:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011749376/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829126.900876:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829126.900880:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829126.900883:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829126.900887:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4530 00000100:00080000:0.0:1787829126.900891:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829126.900895:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829126.900900:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787829126.900908:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829126.900915:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011749504/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829126.900928:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829126.900931:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829126.900933:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829126.900937:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4530 00000100:00100000:2.0:1787829126.900941:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829126/real 1787829126] req@ffff907d90847480 x1874670011749248/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829126.900942:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829126.900952:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829126.900959:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829126.900961:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011749248/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829126.900972:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787829126.900979:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011749632/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829126.900979:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829126.900989:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011749248:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829126.901019:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011749376:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829126.901045:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829126.901055:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829126/real 1787829126] req@ffff907d90846940 x1874670011749376/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829126.901064:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011749376/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829126.901069:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829126.901072:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829126.901074:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011749376:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829126.901087:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011749504:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829126.901101:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829126.901108:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829126/real 1787829126] req@ffff907d90844f00 x1874670011749504/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829126.901117:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011749504/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829126.901124:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829126.901126:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829126.901128:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011749504:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829126.901132:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011749632:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829126.901145:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829126.901157:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829126/real 1787829126] req@ffff907d90845a40 x1874670011749632/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829126.901169:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011749632/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829126.901179:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829126.901183:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829126.901189:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011749632:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829127.028714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829127.028780:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829128.052690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829128.052771:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829129.076731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829129.076763:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829130.100739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829130.100827:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829131.124720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829131.124794:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829132.020709:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829132.020720:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829132.020725:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829132.020736:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4535 00000100:00080000:0.0:1787829132.020750:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829132.020757:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829132.020796:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011749760/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829132.020822:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829132.020825:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829132.020826:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829132.020830:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4535 00000100:00080000:0.0:1787829132.020833:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829132.020836:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829132.020844:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829132.020853:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011749888/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829132.020861:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829132.020864:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829132.020865:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829132.020867:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4535 00000100:00080000:0.0:1787829132.020870:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829132.020873:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829132.020877:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829132.020883:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011750016/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829132.020889:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829132.020892:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829132.020893:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829132.020896:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4535 00000100:00080000:0.0:1787829132.020898:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829132.020900:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829132.020904:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829132.020912:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011750144/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829132.020917:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011749760:192.168.203.104@tcp:250:kworker.0 00000800:00000100:2.0:1787829132.021013:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829132.021070:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829132/real 1787829132] req@ffff907d90846d00 x1874670011749760/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829132.021093:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011749760/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829132.021106:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829132.021111:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829132.021116:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011749760:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829132.021125:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011749888:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829132.021153:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829132.021166:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829132/real 1787829132] req@ffff907d90845680 x1874670011749888/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829132.021180:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011749888/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829132.021190:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829132.021194:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829132.021197:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011749888:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829132.021205:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011750016:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829132.021227:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829132.021239:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829132/real 1787829132] req@ffff907d90844780 x1874670011750016/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829132.021253:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011750016/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829132.021262:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829132.021266:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829132.021270:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011750016:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829132.021303:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011750144:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829132.021326:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829132.021340:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829132/real 1787829132] req@ffff907d908443c0 x1874670011750144/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829132.021377:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011750144/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829132.021387:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829132.021391:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829132.021395:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011750144:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829132.148744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829132.148815:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829133.172708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829133.172776:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829134.196726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829134.196835:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829135.220732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829135.220904:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829136.244682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829136.244768:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829137.140733:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829137.140742:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829137.140746:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829137.140757:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4540 00000100:00080000:2.0:1787829137.140774:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829137.140781:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829137.140823:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011750272/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829137.140853:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829137.140857:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829137.140858:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829137.140861:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4540 00000100:00080000:2.0:1787829137.140864:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829137.140867:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829137.140878:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829137.140890:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011750400/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829137.140892:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011750272:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829137.140918:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829137.140921:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829137.140922:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829137.140928:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4540 00000100:00080000:2.0:1787829137.140931:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829137.140933:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829137.140936:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829137.140949:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011750528/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829137.140956:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829137.140959:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829137.140960:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829137.140964:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4540 00000100:00080000:2.0:1787829137.140968:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829137.140971:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829137.140975:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787829137.140980:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829137.140993:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011750656/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829137.141027:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829137/real 1787829137] req@ffff907d911492c0 x1874670011750272/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829137.141042:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011750272/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829137.141050:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829137.141054:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829137.141057:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011750272:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829137.141076:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011750400:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829137.141093:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829137.141102:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829137/real 1787829137] req@ffff907d91148b40 x1874670011750400/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829137.141110:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011750400/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829137.141115:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829137.141118:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829137.141120:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011750400:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829137.141124:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011750528:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829137.141136:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829137.141142:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829137/real 1787829137] req@ffff907d9114a940 x1874670011750528/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829137.141149:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011750528/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829137.141155:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829137.141156:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829137.141158:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011750528:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829137.141162:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011750656:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829137.141172:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829137.141178:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829137/real 1787829137] req@ffff907d91148f00 x1874670011750656/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829137.141185:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011750656/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829137.141191:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829137.141193:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829137.141194:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011750656:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829137.268698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829137.268753:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829138.292703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829138.292769:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829139.316718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829139.316789:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829140.340679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829140.340775:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829141.364733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829141.364809:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829142.260724:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829142.260734:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829142.260740:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829142.260752:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4546 00000100:00080000:1.0:1787829142.260764:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829142.260771:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829142.260814:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011750784/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829142.260846:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829142.260849:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829142.260851:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829142.260853:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4546 00000100:00080000:1.0:1787829142.260856:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829142.260859:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829142.260869:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829142.260881:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011750912/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829142.260894:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829142.260895:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011750784:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829142.260898:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829142.260905:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829142.260914:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4546 00000100:00080000:1.0:1787829142.260916:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829142.260919:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829142.260923:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829142.260939:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011751040/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829142.260946:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829142.260949:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829142.260950:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829142.260952:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4546 00000100:00080000:1.0:1787829142.260954:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829142.260957:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829142.260960:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829142.260968:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011751168/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829142.260982:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829142.261021:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829142/real 1787829142] req@ffff907d908a2d00 x1874670011750784/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829142.261035:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011750784/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829142.261043:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829142.261047:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829142.261049:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011750784:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829142.261070:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011750912:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829142.261086:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829142.261095:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829142/real 1787829142] req@ffff907d908a1e00 x1874670011750912/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829142.261102:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011750912/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829142.261108:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829142.261111:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829142.261113:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011750912:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829142.261117:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011751040:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829142.261130:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829142.261136:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829142/real 1787829142] req@ffff907d908a21c0 x1874670011751040/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829142.261144:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011751040/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829142.261149:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829142.261151:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829142.261153:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011751040:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829142.261157:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011751168:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829142.261167:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829142.261172:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829142/real 1787829142] req@ffff907d908a3840 x1874670011751168/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829142.261179:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011751168/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829142.261184:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829142.261186:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829142.261188:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011751168:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829142.388682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829142.388783:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829143.412709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829143.412751:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829144.436693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829144.436739:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829145.460684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829145.460728:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829146.484673:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829146.484727:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829147.380674:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829147.380682:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829147.380686:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829147.380694:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4551 00000100:00080000:1.0:1787829147.380707:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829147.380712:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829147.380741:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011751296/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829147.380761:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829147.380764:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829147.380766:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829147.380768:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4551 00000100:00080000:1.0:1787829147.380770:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829147.380773:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829147.380780:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829147.380790:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011751424/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829147.380799:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829147.380802:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011751296:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829147.380803:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829147.380805:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829147.380812:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4551 00000100:00080000:1.0:1787829147.380815:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829147.380817:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829147.380820:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829147.380829:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011751552/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829147.380840:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829147.380842:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829147.380843:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829147.380845:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4551 00000100:00080000:1.0:1787829147.380848:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829147.380850:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829147.380853:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829147.380860:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011751680/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829147.380872:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829147.380907:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829147/real 1787829147] req@ffff907d908a0000 x1874670011751296/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829147.380920:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011751296/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829147.380928:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829147.380931:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829147.380941:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011751296:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829147.380958:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011751424:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829147.380974:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829147.380983:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829147/real 1787829147] req@ffff907d908a2940 x1874670011751424/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829147.380991:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011751424/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829147.380997:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829147.380999:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829147.381001:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011751424:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829147.381005:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011751552:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829147.381016:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829147.381021:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829147/real 1787829147] req@ffff907d908a1680 x1874670011751552/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829147.381033:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011751552/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829147.381039:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829147.381040:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829147.381042:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011751552:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829147.381046:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011751680:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829147.381055:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829147.381061:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829147/real 1787829147] req@ffff907d908a2580 x1874670011751680/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829147.381068:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011751680/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829147.381073:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829147.381075:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829147.381077:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011751680:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829147.508680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829147.508729:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829148.532786:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829148.532829:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829149.556699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829149.556763:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829150.580778:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829150.580810:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829151.604707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829151.604786:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829152.500759:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829152.500767:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829152.500771:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829152.500779:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4556 00000100:00080000:2.0:1787829152.500799:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829152.500806:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829152.500839:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011751808/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829152.500867:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829152.500870:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829152.500871:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829152.500874:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4556 00000100:00080000:2.0:1787829152.500876:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829152.500879:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829152.500887:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829152.500899:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011751936/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829152.500906:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829152.500909:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829152.500911:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829152.500913:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4556 00000100:00080000:2.0:1787829152.500915:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829152.500917:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829152.500920:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829152.500934:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011751808:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829152.500934:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011752064/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829152.500956:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829152.500959:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829152.500960:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829152.500964:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4556 00000100:00080000:2.0:1787829152.500966:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829152.500971:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829152.500974:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829152.500981:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011752192/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829152.501033:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829152.501078:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829152/real 1787829152] req@ffff907d91148f00 x1874670011751808/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829152.501095:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011751808/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829152.501105:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829152.501115:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829152.501119:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011751808:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829152.501126:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011751936:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829152.501151:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829152.501166:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829152/real 1787829152] req@ffff907d9114a940 x1874670011751936/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829152.501188:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011751936/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829152.501195:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829152.501197:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829152.501200:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011751936:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829152.501226:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011752064:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829152.501244:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829152.501253:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829152/real 1787829152] req@ffff907d91148b40 x1874670011752064/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829152.501261:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011752064/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829152.501266:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829152.501269:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829152.501271:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011752064:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829152.501275:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011752192:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829152.501286:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829152.501292:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829152/real 1787829152] req@ffff907d911492c0 x1874670011752192/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829152.501299:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011752192/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829152.501305:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829152.501307:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829152.501309:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011752192:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829152.628713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829152.628768:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829153.652728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829153.652809:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829154.676719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829154.676789:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829155.700722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829155.700779:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829156.724708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829156.724771:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829157.620725:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829157.620733:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829157.620737:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829157.620745:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4561 00000100:00080000:0.0:1787829157.620754:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829157.620760:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829157.620793:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011752320/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829157.620815:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829157.620819:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829157.620821:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829157.620826:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4561 00000100:00080000:0.0:1787829157.620830:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829157.620834:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829157.620844:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829157.620857:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011752448/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829157.620863:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011752320:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829157.620868:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829157.620871:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829157.620872:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829157.620876:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4561 00000100:00080000:0.0:1787829157.620879:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829157.620882:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829157.620887:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829157.620895:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011752576/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829157.620903:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829157.620906:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829157.620908:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829157.620911:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4561 00000100:00080000:0.0:1787829157.620913:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829157.620916:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829157.620920:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787829157.620923:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829157.620929:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011752704/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829157.620963:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829157/real 1787829157] req@ffff907d90845e00 x1874670011752320/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829157.620981:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011752320/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829157.620989:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829157.620992:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829157.620994:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011752320:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829157.621013:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011752448:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829157.621031:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829157.621040:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829157/real 1787829157] req@ffff907d90844000 x1874670011752448/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829157.621048:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011752448/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829157.621055:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829157.621059:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829157.621062:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011752448:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829157.621069:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011752576:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829157.621090:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829157.621103:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829157/real 1787829157] req@ffff907d908452c0 x1874670011752576/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829157.621115:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011752576/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829157.621121:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829157.621123:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829157.621125:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011752576:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829157.621129:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011752704:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829157.621144:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829157.621151:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829157/real 1787829157] req@ffff907d90844b40 x1874670011752704/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829157.621158:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011752704/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829157.621162:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829157.621165:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829157.621166:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011752704:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829157.748688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829157.748721:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829158.772718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829158.772771:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829159.796689:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829159.796762:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829160.820697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829160.820750:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829161.844674:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829161.844741:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829162.740669:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829162.740677:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829162.740681:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829162.740688:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4566 00000100:00080000:0.0:1787829162.740697:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829162.740703:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829162.740732:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011752832/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829162.740757:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829162.740762:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829162.740764:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829162.740768:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4566 00000100:00080000:0.0:1787829162.740773:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829162.740777:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829162.740788:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829162.740803:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011752960/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829162.740810:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011752832:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829162.740814:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829162.740819:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829162.740821:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829162.740824:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4566 00000100:00080000:0.0:1787829162.740827:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829162.740829:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829162.740833:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829162.740851:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011753088/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829162.740859:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829162.740862:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829162.740863:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829162.740866:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4566 00000100:00080000:0.0:1787829162.740868:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829162.740870:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829162.740874:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787829162.740875:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829162.740885:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011753216/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829162.740909:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829162/real 1787829162] req@ffff907d908470c0 x1874670011752832/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829162.740924:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011752832/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829162.740932:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829162.740935:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829162.740939:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011752832:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829162.740957:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011752960:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829162.740975:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829162.740983:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829162/real 1787829162] req@ffff907d90846580 x1874670011752960/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829162.740991:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011752960/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829162.740997:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829162.741000:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829162.741002:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011752960:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829162.741005:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011753088:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829162.741017:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829162.741023:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829162/real 1787829162] req@ffff907d90847c00 x1874670011753088/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829162.741034:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011753088/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829162.741044:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829162.741047:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829162.741050:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011753088:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829162.741054:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011753216:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829162.741066:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829162.741074:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829162/real 1787829162] req@ffff907d90844b40 x1874670011753216/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829162.741084:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011753216/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829162.741094:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829162.741098:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829162.741102:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011753216:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829162.868693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829162.868763:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829163.892682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829163.892712:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829164.916674:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829164.916742:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829165.940703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829165.940759:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829166.964701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829166.964770:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829167.860706:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829167.860716:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829167.860721:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829167.860732:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4571 00000100:00080000:0.0:1787829167.860743:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829167.860751:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829167.860789:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011753344/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829167.860813:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829167.860817:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829167.860819:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829167.860822:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4571 00000100:00080000:0.0:1787829167.860826:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829167.860831:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829167.860842:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829167.860869:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011753472/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829167.860879:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829167.860884:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829167.860887:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829167.860887:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011753344:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829167.860895:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4571 00000100:00080000:0.0:1787829167.860901:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829167.860904:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829167.860909:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829167.860925:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011753600/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829167.860938:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829167.860943:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829167.860946:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829167.860951:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4571 00000100:00080000:0.0:1787829167.860957:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787829167.860962:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787829167.860968:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829167.860977:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829167.860993:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011753728/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829167.861011:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829167/real 1787829167] req@ffff907d908452c0 x1874670011753344/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829167.861030:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011753344/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829167.861040:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829167.861045:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829167.861050:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011753344:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829167.861072:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011753472:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829167.861092:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829167.861102:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829167/real 1787829167] req@ffff907d90844000 x1874670011753472/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829167.861113:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011753472/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829167.861119:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829167.861122:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829167.861125:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011753472:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829167.861130:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011753600:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829167.861144:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829167.861151:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829167/real 1787829167] req@ffff907d90845e00 x1874670011753600/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829167.861160:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011753600/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829167.861166:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829167.861168:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829167.861171:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011753600:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829167.861175:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011753728:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829167.861186:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829167.861193:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829167/real 1787829167] req@ffff907d908443c0 x1874670011753728/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829167.861202:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011753728/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829167.861209:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829167.861211:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829167.861213:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011753728:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829167.988777:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829167.988912:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829169.012777:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829169.012980:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829170.036769:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829170.036831:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829171.060695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829171.060780:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829172.084769:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829172.084837:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829172.980732:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829172.980741:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829172.980745:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829172.980778:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4576 00000100:00080000:1.0:1787829172.980797:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829172.980807:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829172.980861:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011753856/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829172.980887:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829172.980890:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829172.980891:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829172.980894:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4576 00000100:00080000:1.0:1787829172.980897:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829172.980900:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829172.980907:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829172.980916:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011753984/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829172.980929:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829172.980931:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829172.980933:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829172.980936:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4576 00000100:00080000:1.0:1787829172.980938:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829172.980941:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011753856:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829172.980943:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829172.980951:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829172.980961:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011754112/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829172.980967:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829172.980970:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829172.980972:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829172.980974:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4576 00000100:00080000:1.0:1787829172.980976:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829172.980979:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829172.980983:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829172.980989:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011754240/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829172.981007:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829172.981041:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829172/real 1787829172] req@ffff907d908a12c0 x1874670011753856/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829172.981054:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011753856/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829172.981061:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829172.981064:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829172.981067:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011753856:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829172.981072:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011753984:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829172.981089:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829172.981097:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829172/real 1787829172] req@ffff907d908a0f00 x1874670011753984/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829172.981105:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011753984/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829172.981111:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829172.981113:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829172.981115:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011753984:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829172.981132:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011754112:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829172.981144:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829172.981151:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829172/real 1787829172] req@ffff907d908a2580 x1874670011754112/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829172.981158:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011754112/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829172.981168:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829172.981170:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829172.981172:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011754112:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829172.981176:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011754240:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829172.981186:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829172.981192:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829172/real 1787829172] req@ffff907d908a1680 x1874670011754240/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829172.981199:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011754240/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829172.981204:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829172.981206:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829172.981208:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011754240:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829173.108708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829173.108779:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829174.132694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829174.132766:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829175.156733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829175.156797:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829176.180686:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829176.180793:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829177.204682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829177.204719:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829178.100676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829178.100684:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829178.100687:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829178.100695:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4581 00000100:00080000:1.0:1787829178.100704:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829178.100710:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829178.100738:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011754368/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829178.100760:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829178.100763:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829178.100764:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829178.100767:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4581 00000100:00080000:1.0:1787829178.100769:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829178.100771:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829178.100778:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829178.100787:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011754496/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829178.100794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829178.100797:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829178.100799:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829178.100801:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4581 00000100:00080000:1.0:1787829178.100803:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829178.100807:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011754368:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829178.100808:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829178.100816:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829178.100825:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011754624/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829178.100832:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829178.100836:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829178.100837:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829178.100840:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4581 00000100:00080000:1.0:1787829178.100842:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829178.100844:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829178.100847:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829178.100854:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011754752/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829178.100900:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011754496:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829178.100941:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011754624:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829178.100957:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011754752:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829178.228702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829178.228753:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829179.252692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829179.252764:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829180.276694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829180.276738:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787829181.172839:0:2337:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:3.0:1787829181.172881:0:2337:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787829181.172898:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787829181.172938:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787829181.172948:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787829181.172958:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:2.0:1787829181.172987:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829178/real 1787829181] req@ffff907d908a3840 x1874670011754624/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829181.173020:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011754624/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829181.173033:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829181.173039:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829181.173044:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011754624:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829181.173062:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829178/real 1787829181] req@ffff907d908a21c0 x1874670011754752/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829181.173077:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011754752/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829181.173086:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829181.173089:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829181.173093:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011754752:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829181.173100:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829178/real 1787829181] req@ffff907d908a2940 x1874670011754368/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829181.173114:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011754368/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829181.173123:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829181.173127:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829181.173131:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011754368:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829181.173138:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829178/real 1787829181] req@ffff907d908a0000 x1874670011754496/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829181.173148:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011754496/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829181.173156:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829181.173160:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829181.173163:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011754496:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829181.300680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829181.300741:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829182.324715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829182.324753:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829183.220697:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829183.220710:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829183.220715:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829183.220727:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4586 00000100:00080000:1.0:1787829183.220740:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829183.220750:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829183.220795:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011754880/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829183.220828:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829183.220832:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829183.220842:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829183.220850:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4586 00000100:00080000:1.0:1787829183.220855:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829183.220861:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829183.220873:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829183.220889:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011755008/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829183.220899:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011754880:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829183.220910:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829183.220916:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829183.220920:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829183.220927:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4586 00000100:00080000:1.0:1787829183.220932:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829183.220936:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829183.220944:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829183.220959:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011755136/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829183.220969:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829183.220973:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829183.220975:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787829183.220984:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787829183.220986:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4586 00000100:00080000:1.0:1787829183.220994:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829183.221000:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829183.221007:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829183.221020:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011755264/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829183.221046:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829183/real 1787829183] req@ffff907d908a1e00 x1874670011754880/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829183.221069:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011754880/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829183.221087:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829183.221092:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829183.221097:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011754880:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829183.221142:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011755008:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829183.221170:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829183.221185:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829183/real 1787829183] req@ffff907d908a2d00 x1874670011755008/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829183.221199:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011755008/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829183.221209:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829183.221212:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829183.221216:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011755008:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829183.221223:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011755136:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829183.221834:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829183.221857:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829183/real 1787829183] req@ffff907d908a0000 x1874670011755136/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829183.221871:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011755136/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829183.221880:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829183.221883:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829183.221886:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011755136:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829183.221894:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011755264:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829183.221924:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829183.221936:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829183/real 1787829183] req@ffff907d908a2940 x1874670011755264/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829183.221949:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011755264/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829183.221964:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829183.221968:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829183.221972:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011755264:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829183.348730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829183.348798:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829184.372677:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829184.372720:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829185.396721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829185.396799:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829186.420710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829186.420771:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829187.444761:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829187.444842:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829188.340741:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829188.340749:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829188.340753:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829188.340761:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4592 00000100:00080000:0.0:1787829188.340770:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829188.340776:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829188.340816:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011755392/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829188.340839:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829188.340845:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829188.340846:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829188.340849:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4592 00000100:00080000:0.0:1787829188.340851:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829188.340854:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829188.340862:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829188.340871:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011755520/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829188.340878:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829188.340881:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829188.340884:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829188.340885:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011755392:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829188.340891:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4592 00000100:00080000:0.0:1787829188.340894:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829188.340897:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829188.340900:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829188.340908:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011755648/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829188.340915:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829188.340918:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829188.340919:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829188.340921:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4592 00000100:00080000:0.0:1787829188.340923:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829188.340926:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829188.340929:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829188.340934:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011755776/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829188.340952:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829188.340993:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829188/real 1787829188] req@ffff907d90844780 x1874670011755392/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829188.341008:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011755392/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829188.341015:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829188.341018:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829188.341021:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011755392:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829188.341038:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011755520:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829188.341055:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829188.341063:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829188/real 1787829188] req@ffff907d90845680 x1874670011755520/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829188.341071:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011755520/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829188.341080:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829188.341083:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829188.341085:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011755520:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829188.341088:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011755648:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829188.341099:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829188.341104:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829188/real 1787829188] req@ffff907d90846d00 x1874670011755648/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829188.341111:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011755648/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829188.341117:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829188.341118:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829188.341120:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011755648:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829188.341124:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011755776:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829188.341133:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829188.341138:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829188/real 1787829188] req@ffff907d90845a40 x1874670011755776/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829188.341145:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011755776/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829188.341150:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829188.341151:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829188.341154:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011755776:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829188.468678:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829188.468728:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829189.492684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829189.492710:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829190.516726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829190.516791:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829191.540710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829191.540731:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829192.564759:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829192.564809:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829193.460739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829193.460752:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829193.460756:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829193.460765:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4597 00000100:00080000:2.0:1787829193.460775:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829193.460781:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829193.460814:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011755904/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829193.460839:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829193.460843:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829193.460844:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829193.460848:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4597 00000100:00080000:2.0:1787829193.460850:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829193.460853:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829193.460861:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829193.460872:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011756032/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829193.460881:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829193.460887:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011755904:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829193.460887:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829193.460889:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829193.460894:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4597 00000100:00080000:2.0:1787829193.460897:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829193.460913:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829193.460916:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829193.460925:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011756160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829193.460933:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829193.460935:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829193.460937:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829193.460940:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4597 00000100:00080000:2.0:1787829193.460942:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829193.460944:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829193.460948:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829193.460955:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011756288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829193.460971:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829193.461015:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829193/real 1787829193] req@ffff907d9114a580 x1874670011755904/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829193.461031:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011755904/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829193.461040:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829193.461043:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829193.461046:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011755904:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829193.461064:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011756032:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829193.461081:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829193.461091:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829193/real 1787829193] req@ffff907d9114ad00 x1874670011756032/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829193.461100:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011756032/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829193.461106:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829193.461108:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829193.461111:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011756032:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829193.461115:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011756160:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829193.461127:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829193.461133:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829193/real 1787829193] req@ffff907d91149a40 x1874670011756160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829193.461141:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011756160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829193.461152:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829193.461155:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829193.461157:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011756160:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829193.461161:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011756288:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829193.461171:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829193.461196:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829193/real 1787829193] req@ffff907d9114b840 x1874670011756288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829193.461204:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011756288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829193.461220:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829193.461223:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829193.461225:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011756288:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829193.588731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829193.588855:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829194.612687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829194.612747:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829195.636716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829195.636879:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829196.660698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829196.660781:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829197.684682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829197.684738:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829198.580679:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829198.580687:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829198.580691:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829198.580699:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4602 00000100:00080000:2.0:1787829198.580708:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829198.580714:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829198.580746:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011756416/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829198.580769:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829198.580774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829198.580780:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829198.580785:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4602 00000100:00080000:2.0:1787829198.580789:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829198.580794:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829198.580804:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829198.580817:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011756416:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829198.580822:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011756544/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829198.580834:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829198.580837:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829198.580838:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829198.580841:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4602 00000100:00080000:2.0:1787829198.580843:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829198.580846:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829198.580849:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829198.580857:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011756672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829198.580863:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829198.580866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829198.580867:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829198.580874:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4602 00000100:00080000:2.0:1787829198.580878:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829198.580880:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829198.580883:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829198.580892:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011756800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829198.580932:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829198.580967:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829198/real 1787829198] req@ffff907d9114b480 x1874670011756416/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829198.580982:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011756416/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829198.580990:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829198.580993:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829198.580996:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011756416:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829198.581017:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011756544:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829198.581038:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829198.581051:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829198/real 1787829198] req@ffff907d91149680 x1874670011756544/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829198.581063:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011756544/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829198.581069:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829198.581072:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829198.581075:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011756544:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829198.581080:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011756672:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829198.581094:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829198.581101:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829198/real 1787829198] req@ffff907d9114bc00 x1874670011756672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829198.581111:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011756672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829198.581117:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829198.581119:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829198.581122:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011756672:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829198.581127:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011756800:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829198.581172:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829198.581182:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829198/real 1787829198] req@ffff907d911483c0 x1874670011756800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829198.581191:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011756800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829198.581199:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829198.581205:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829198.581210:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011756800:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829198.708688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829198.708706:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829199.732721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829199.732807:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829200.756672:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829200.756782:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829201.780726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829201.780779:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829202.804676:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829202.804737:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829203.700719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829203.700748:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829203.700752:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829203.700763:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4607 00000100:00080000:2.0:1787829203.700773:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829203.700780:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829203.700817:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011756928/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829203.700843:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829203.700847:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829203.700849:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829203.700852:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4607 00000100:00080000:2.0:1787829203.700856:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829203.700860:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829203.700870:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829203.700882:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011757056/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829203.700888:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011756928:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829203.700910:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829203.700916:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829203.700919:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829203.700926:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4607 00000100:00080000:2.0:1787829203.700948:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829203.700953:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829203.700960:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787829203.700977:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829203.700979:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011757184/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829203.700990:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829203.700994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829203.700995:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829203.700998:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4607 00000100:00080000:2.0:1787829203.701002:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829203.701021:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829203.701027:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829203.701038:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829203/real 1787829203] req@ffff907d911483c0 x1874670011756928/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829203.701042:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011757312/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829203.701089:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011756928/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829203.701122:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829203.701128:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829203.701134:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011756928:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829203.701174:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011757056:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829203.701200:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829203.701212:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829203/real 1787829203] req@ffff907d9114bc00 x1874670011757056/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829203.701229:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011757056/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829203.701266:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829203.701271:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829203.701277:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011757056:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829203.701286:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011757184:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829203.701315:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829203.701331:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829203/real 1787829203] req@ffff907d91149680 x1874670011757184/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829203.701348:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011757184/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829203.701359:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829203.701364:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829203.701368:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011757184:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829203.701375:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011757312:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829203.701396:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829203.701406:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829203/real 1787829203] req@ffff907d9114b480 x1874670011757312/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829203.701416:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011757312/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829203.701423:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829203.701425:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829203.701429:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011757312:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787829203.828728:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829203.828749:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787829204.852746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829204.852832:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829205.876716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829205.876753:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829206.900716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829206.900756:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829207.924713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829207.924786:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829208.820723:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829208.820731:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829208.820734:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829208.820742:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4612 00000100:00080000:3.0:1787829208.820752:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829208.820757:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829208.820788:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011757440/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829208.820811:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829208.820814:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829208.820815:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829208.820818:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4612 00000100:00080000:3.0:1787829208.820820:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829208.820823:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829208.820830:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829208.820841:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011757568/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829208.820848:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829208.820851:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829208.820852:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829208.820855:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4612 00000100:00080000:3.0:1787829208.820858:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829208.820860:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011757440:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829208.820862:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829208.820871:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829208.820879:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011757696/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829208.820887:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829208.820890:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829208.820891:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829208.820894:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4612 00000100:00080000:3.0:1787829208.820896:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829208.820898:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829208.820901:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829208.820909:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011757824/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829208.820920:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829208.820961:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829208/real 1787829208] req@ffff907da562f480 x1874670011757440/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829208.820982:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011757440/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829208.820990:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829208.820994:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829208.820997:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011757440:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829208.821003:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011757568:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829208.821022:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829208.821031:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829208/real 1787829208] req@ffff907da562c780 x1874670011757568/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829208.821039:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011757568/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829208.821045:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829208.821047:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829208.821049:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011757568:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829208.821084:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011757696:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829208.821103:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829208.821111:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829208/real 1787829208] req@ffff907da562ed00 x1874670011757696/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829208.821118:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011757696/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829208.821124:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829208.821126:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829208.821128:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011757696:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829208.821132:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011757824:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829208.821142:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829208.821148:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829208/real 1787829208] req@ffff907da562cf00 x1874670011757824/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829208.821155:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011757824/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829208.821160:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829208.821162:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829208.821164:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011757824:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829208.948701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829208.948765:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829209.972710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829209.972779:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829210.996695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829210.996741:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829212.020701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829212.020764:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829213.044715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829213.044881:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829213.940693:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829213.940704:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829213.940708:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829213.940720:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4617 00000100:00080000:3.0:1787829213.940732:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829213.940741:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829213.940782:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011757952/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829213.940814:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829213.940819:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829213.940821:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829213.940825:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4617 00000100:00080000:3.0:1787829213.940829:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829213.940832:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829213.940843:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829213.940879:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011758080/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829213.940895:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829213.940900:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829213.940902:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829213.940913:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4617 00000100:00100000:1.0:1787829213.940916:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011757952:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829213.940918:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829213.940932:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829213.940941:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829213.940958:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011758208/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829213.940971:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829213.940976:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829213.940979:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829213.940983:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4617 00000100:00080000:3.0:1787829213.940988:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829213.940992:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829213.940999:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787829213.941015:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829213.941021:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011758336/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829213.941116:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829213/real 1787829213] req@ffff907da562d680 x1874670011757952/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829213.941147:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011757952/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829213.941159:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829213.941163:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829213.941168:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011757952:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829213.941176:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011758080:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829213.941203:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829213.941216:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829213/real 1787829213] req@ffff907da562cf00 x1874670011758080/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829213.941228:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011758080/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829213.941237:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829213.941241:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829213.941244:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011758080:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829213.941271:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011758208:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829213.941290:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829213.941300:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829213/real 1787829213] req@ffff907da562ed00 x1874670011758208/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829213.941312:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011758208/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829213.941327:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829213.941331:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829213.941334:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011758208:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829213.941340:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011758336:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829213.941355:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829213.941364:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829213/real 1787829213] req@ffff907da562c780 x1874670011758336/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829213.941376:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011758336/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829213.941385:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829213.941388:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829213.941391:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011758336:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829214.068690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829214.068744:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829215.092727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829215.092833:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829216.116687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829216.116760:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829217.140678:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829217.140749:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829218.164693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829218.164740:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829219.060673:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829219.060682:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829219.060685:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829219.060695:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4622 00000100:00080000:3.0:1787829219.060706:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829219.060714:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829219.060750:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011758464/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829219.060778:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829219.060781:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829219.060782:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829219.060785:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4622 00000100:00080000:3.0:1787829219.060788:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829219.060791:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829219.060799:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829219.060809:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011758592/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829219.060816:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829219.060819:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829219.060820:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829219.060823:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4622 00000100:00080000:3.0:1787829219.060825:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829219.060831:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829219.060833:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011758464:192.168.203.104@tcp:250:kworker.0 00000008:00100000:3.0:1787829219.060835:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829219.060856:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011758720/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829219.060863:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829219.060866:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829219.060867:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829219.060869:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4622 00000100:00080000:3.0:1787829219.060872:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829219.060874:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829219.060876:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829219.060885:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011758848/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829219.060927:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829219.060968:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829219/real 1787829219] req@ffff907da562f480 x1874670011758464/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829219.060988:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011758464/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829219.061012:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829219.061017:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829219.061021:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011758464:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829219.061030:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011758592:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829219.061066:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829219.061080:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829219/real 1787829219] req@ffff907da562e580 x1874670011758592/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829219.061093:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011758592/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829219.061102:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829219.061105:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829219.061109:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011758592:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829219.061144:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011758720:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829219.061169:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829219.061190:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829219/real 1787829219] req@ffff907da562d2c0 x1874670011758720/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829219.061205:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011758720/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829219.061214:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829219.061217:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829219.061220:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011758720:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829219.061227:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011758848:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829219.061249:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829219.061262:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829219/real 1787829219] req@ffff907da562fc00 x1874670011758848/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829219.061275:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011758848/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829219.061283:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829219.061287:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829219.061290:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011758848:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829219.188679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829219.188747:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829220.212688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829220.212734:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829221.236718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829221.236754:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829222.260691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829222.260760:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829223.284720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829223.284836:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829224.180722:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829224.180735:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829224.180740:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829224.180754:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4627 00000100:00080000:0.0:1787829224.180768:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829224.180776:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829224.180820:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011758976/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829224.180857:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829224.180862:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829224.180864:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829224.180868:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4627 00000100:00080000:0.0:1787829224.180872:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829224.180875:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829224.180888:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829224.180900:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011759104/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829224.180913:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829224.180917:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011758976:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829224.180918:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829224.180922:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829224.180928:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4627 00000100:00080000:0.0:1787829224.180932:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829224.180936:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829224.180941:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829224.180955:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011759232/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829224.180979:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829224.180983:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829224.180986:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829224.180990:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4627 00000100:00080000:0.0:1787829224.180994:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829224.180999:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829224.181004:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787829224.181010:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829224.181020:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011759360/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829224.181057:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829224/real 1787829224] req@ffff907d90844f00 x1874670011758976/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829224.181073:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011758976/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829224.181083:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829224.181085:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829224.181089:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011758976:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829224.181111:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011759104:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829224.181127:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829224.181135:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829224/real 1787829224] req@ffff907d90846940 x1874670011759104/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829224.181143:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011759104/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829224.181154:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829224.181156:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829224.181159:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011759104:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829224.181162:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011759232:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829224.181182:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829224.181190:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829224/real 1787829224] req@ffff907d90847480 x1874670011759232/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829224.181198:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011759232/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829224.181204:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829224.181206:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829224.181208:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011759232:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829224.181211:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011759360:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829224.181222:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829224.181228:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829224/real 1787829224] req@ffff907d90845a40 x1874670011759360/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829224.181235:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011759360/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829224.181241:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829224.181242:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829224.181245:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011759360:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829224.308734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829224.308851:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829225.332708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829225.332763:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829226.356697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829226.356711:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829227.380729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829227.380781:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829228.404751:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829228.404799:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829229.300733:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829229.300743:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829229.300748:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829229.300764:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4633 00000100:00080000:1.0:1787829229.300776:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829229.300784:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829229.300827:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011759488/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829229.300859:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829229.300862:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829229.300863:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829229.300866:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4633 00000100:00080000:1.0:1787829229.300869:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829229.300872:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829229.300882:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829229.300892:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011759616/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829229.300901:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829229.300903:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829229.300906:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829229.300909:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011759488:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829229.300911:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4633 00000100:00080000:1.0:1787829229.300917:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829229.300921:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829229.300938:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829229.300949:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011759744/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829229.300956:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829229.300959:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829229.300960:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829229.300963:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4633 00000100:00080000:1.0:1787829229.300967:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829229.300971:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829229.300976:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829229.300989:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011759872/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829229.301001:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829229.301045:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829229/real 1787829229] req@ffff907d908a21c0 x1874670011759488/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829229.301060:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011759488/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829229.301070:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829229.301073:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829229.301076:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011759488:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829229.301101:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011759616:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829229.301118:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829229.301127:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829229/real 1787829229] req@ffff907d908a3840 x1874670011759616/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829229.301136:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011759616/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829229.301142:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829229.301144:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829229.301146:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011759616:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829229.301150:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011759744:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829229.301161:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829229.301167:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829229/real 1787829229] req@ffff907d908a1680 x1874670011759744/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829229.301174:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011759744/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829229.301180:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829229.301182:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829229.301183:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011759744:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829229.301187:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011759872:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829229.301197:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829229.301203:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829229/real 1787829229] req@ffff907d908a2580 x1874670011759872/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829229.301211:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011759872/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829229.301217:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829229.301220:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829229.301223:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011759872:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829229.428709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829229.428755:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829230.452714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829230.452824:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829231.476678:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829231.476763:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829232.500691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829232.500738:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829233.524702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829233.524737:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829234.420677:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829234.420686:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829234.420690:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829234.420698:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4638 00000100:00080000:1.0:1787829234.420707:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829234.420714:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829234.420745:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011760000/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829234.420770:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829234.420773:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829234.420775:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829234.420778:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4638 00000100:00080000:1.0:1787829234.420782:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829234.420785:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829234.420794:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829234.420801:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011760000:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829234.420808:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011760128/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829234.420817:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829234.420820:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829234.420821:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829234.420825:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4638 00000100:00080000:1.0:1787829234.420827:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829234.420830:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829234.420833:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829234.420843:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011760256/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829234.420851:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829234.420854:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829234.420856:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829234.420860:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4638 00000800:00000100:0.0:1787829234.420864:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787829234.420866:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829234.420871:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829234.420876:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829234.420884:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011760384/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829234.420915:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829234/real 1787829234] req@ffff907d908a0f00 x1874670011760000/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829234.420931:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011760000/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829234.420939:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829234.420941:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829234.420944:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011760000:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829234.420963:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011760128:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829234.420981:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829234.420989:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829234/real 1787829234] req@ffff907d908a12c0 x1874670011760128/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829234.420998:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011760128/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829234.421004:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829234.421007:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829234.421011:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011760128:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829234.421015:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011760256:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829234.421026:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829234.421033:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829234/real 1787829234] req@ffff907d908a2580 x1874670011760256/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829234.421040:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011760256/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829234.421045:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829234.421047:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829234.421049:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011760256:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829234.421053:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011760384:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829234.421062:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829234.421067:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829234/real 1787829234] req@ffff907d908a1680 x1874670011760384/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829234.421075:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011760384/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829234.421080:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829234.421081:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829234.421083:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011760384:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829234.548696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829234.548756:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829235.572701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829235.572760:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829236.596694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829236.596746:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829237.620707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829237.620766:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829238.644693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829238.644753:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829239.540694:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829239.540703:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829239.540707:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829239.540718:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4643 00000100:00080000:0.0:1787829239.540730:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829239.540737:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829239.540775:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011760512/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829239.540799:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829239.540801:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829239.540803:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829239.540806:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4643 00000100:00080000:0.0:1787829239.540809:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829239.540812:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829239.540819:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829239.540829:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011760640/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829239.540837:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829239.540840:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829239.540841:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829239.540846:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4643 00000100:00100000:2.0:1787829239.540848:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011760512:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829239.540849:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829239.540867:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829239.540872:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829239.540883:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011760768/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829239.540891:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829239.540893:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829239.540895:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829239.540897:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4643 00000100:00080000:0.0:1787829239.540902:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829239.540905:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829239.540908:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829239.540917:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011760896/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829239.540925:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829239.540966:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829239/real 1787829239] req@ffff907d90846d00 x1874670011760512/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829239.540985:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011760512/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829239.540997:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829239.541001:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829239.541006:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011760512:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829239.541027:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011760640:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829239.541049:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829239.541061:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829239/real 1787829239] req@ffff907d90845680 x1874670011760640/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829239.541081:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011760640/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829239.541090:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829239.541093:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829239.541097:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011760640:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829239.541103:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011760768:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829239.541121:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829239.541130:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829239/real 1787829239] req@ffff907d90844780 x1874670011760768/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829239.541143:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011760768/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829239.541151:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829239.541154:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829239.541158:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011760768:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829239.541164:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011760896:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829239.541179:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829239.541189:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829239/real 1787829239] req@ffff907d908443c0 x1874670011760896/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829239.541201:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011760896/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829239.541216:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829239.541220:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829239.541223:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011760896:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829239.668686:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829239.668743:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829240.692681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829240.692703:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829241.716682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829241.716730:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829242.740686:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829242.740812:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829243.764676:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829243.764688:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829244.660722:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829244.660730:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829244.660734:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829244.660742:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4648 00000100:00080000:2.0:1787829244.660751:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829244.660757:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829244.660799:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011761024/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829244.660830:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829244.660835:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829244.660837:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829244.660841:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4648 00000100:00080000:2.0:1787829244.660854:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829244.660859:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829244.660868:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829244.660871:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011761024:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829244.660879:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011761152/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829244.660889:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829244.660893:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829244.660894:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829244.660898:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4648 00000100:00080000:2.0:1787829244.660901:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829244.660904:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829244.660919:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829244.660927:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011761280/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829244.660936:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829244.660939:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829244.660940:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829244.660944:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4648 00000100:00080000:2.0:1787829244.660947:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829244.660950:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829244.660954:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829244.660963:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011761408/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829244.660992:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011761152:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829244.661025:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011761280:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829244.661050:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011761408:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829244.788680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829244.788740:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829245.812690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829245.812746:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829246.836715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829246.836773:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787829247.732868:0:2338:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:3.0:1787829247.732911:0:2338:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787829247.732930:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787829247.732994:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787829247.733007:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787829247.733030:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:0.0:1787829247.733159:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829244/real 1787829247] req@ffff907d91149a40 x1874670011761152/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829247.733199:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011761152/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829247.733237:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829247.733244:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829247.733249:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011761152:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829247.733259:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829244/real 1787829247] req@ffff907d9114ad00 x1874670011761280/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829247.733277:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011761280/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829247.733290:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829247.733311:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829247.733316:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011761280:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829247.733325:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829244/real 1787829247] req@ffff907d9114a580 x1874670011761408/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829247.733342:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011761408/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829247.733352:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829247.733356:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829247.733360:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011761408:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829247.733369:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829244/real 1787829247] req@ffff907d9114b840 x1874670011761024/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829247.733386:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011761024/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829247.733397:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829247.733402:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829247.733422:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011761024:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829247.860701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829247.860811:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829248.884738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829248.884879:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829249.780745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829249.780756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829249.780760:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829249.780770:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4653 00000100:00080000:3.0:1787829249.780794:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829249.780802:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829249.780835:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011761536/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829249.780862:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829249.780866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829249.780868:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829249.780872:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4653 00000100:00080000:3.0:1787829249.780876:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829249.780879:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829249.780889:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829249.780902:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011761664/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829249.780913:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829249.780916:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829249.780918:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829249.780922:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4653 00000100:00080000:3.0:1787829249.780926:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829249.780930:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829249.780935:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829249.780946:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011761792/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829249.780955:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829249.780958:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829249.780959:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829249.780963:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4653 00000100:00080000:3.0:1787829249.780967:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829249.780970:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829249.780975:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829249.780986:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011761920/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829249.781155:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011761536:192.168.203.104@tcp:250:kworker.0 00000800:00000100:0.0:1787829249.781350:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829249.781390:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829249/real 1787829249] req@ffff907da562c000 x1874670011761536/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829249.781416:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011761536/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829249.781426:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829249.781430:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829249.781434:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011761536:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829249.781443:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011761664:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829249.781468:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829249.781498:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829249/real 1787829249] req@ffff907da562e940 x1874670011761664/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829249.781511:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011761664/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829249.781519:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829249.781523:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829249.781526:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011761664:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829249.781532:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011761792:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829249.781555:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829249.781575:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829249/real 1787829249] req@ffff907da562c3c0 x1874670011761792/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829249.781587:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011761792/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829249.781596:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829249.781600:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829249.781640:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011761792:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829249.781647:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011761920:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829249.781671:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829249.781685:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829249/real 1787829249] req@ffff907da562f0c0 x1874670011761920/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829249.781697:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011761920/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829249.781706:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829249.781709:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829249.781713:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011761920:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829249.908728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829249.908827:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829250.932714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829250.932811:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829251.956704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829251.956820:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829252.980739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829252.980823:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829254.004682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829254.004733:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829254.900687:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829254.900696:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829254.900700:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829254.900708:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4658 00000100:00080000:3.0:1787829254.900717:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829254.900722:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829254.900764:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011762048/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829254.900786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829254.900789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829254.900790:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829254.900793:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4658 00000100:00080000:3.0:1787829254.900796:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829254.900798:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829254.900807:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829254.900817:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011762176/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829254.900825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829254.900830:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011762048:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829254.900830:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829254.900834:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829254.900842:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4658 00000100:00080000:3.0:1787829254.900844:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829254.900846:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829254.900850:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829254.900860:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011762304/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829254.900867:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829254.900869:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829254.900871:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829254.900874:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4658 00000100:00080000:3.0:1787829254.900877:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829254.900879:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829254.900882:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829254.900889:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011762432/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829254.900895:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829254.900932:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829254/real 1787829254] req@ffff907da562f840 x1874670011762048/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829254.900945:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011762048/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829254.900953:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829254.900956:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829254.900959:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011762048:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829254.900977:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011762176:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829254.900993:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829254.901001:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829254/real 1787829254] req@ffff907da562e1c0 x1874670011762176/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829254.901009:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011762176/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829254.901015:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829254.901017:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829254.901019:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011762176:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829254.901023:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011762304:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829254.901035:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829254.901040:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829254/real 1787829254] req@ffff907da562cb40 x1874670011762304/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829254.901048:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011762304/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829254.901054:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829254.901055:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829254.901057:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011762304:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829254.901061:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011762432:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829254.901071:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829254.901076:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829254/real 1787829254] req@ffff907da562da40 x1874670011762432/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829254.901083:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011762432/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829254.901088:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829254.901090:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829254.901092:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011762432:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829255.028712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829255.028828:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829256.052673:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829256.052755:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829257.076760:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829257.076811:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787829258.100758:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829258.100828:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829259.124851:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829259.124961:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829260.020849:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829260.020860:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829260.020865:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829260.020879:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4663 00000100:00080000:0.0:1787829260.020891:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829260.020899:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829260.020966:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011762560/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829260.021010:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829260.021016:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829260.021018:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829260.021023:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4663 00000100:00080000:0.0:1787829260.021028:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829260.021034:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829260.021065:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829260.021081:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011762688/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829260.021095:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829260.021099:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829260.021101:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829260.021109:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4663 00000100:00080000:0.0:1787829260.021119:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829260.021120:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011762560:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829260.021134:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829260.021148:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829260.021175:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011762816/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829260.021192:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829260.021197:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829260.021201:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829260.021206:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4663 00000100:00080000:0.0:1787829260.021212:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829260.021217:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829260.021225:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829260.021241:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011762944/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829260.021252:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829260.021332:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829260/real 1787829260] req@ffff907d90845e00 x1874670011762560/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829260.021358:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011762560/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829260.021382:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829260.021388:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829260.021393:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011762560:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829260.021429:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011762688:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829260.021456:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829260.021471:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829260/real 1787829260] req@ffff907d90844000 x1874670011762688/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829260.021497:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011762688/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829260.021517:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829260.021521:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829260.021525:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011762688:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829260.021533:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011762816:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829260.021583:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829260.021662:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829260/real 1787829260] req@ffff907d908452c0 x1874670011762816/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829260.021688:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011762816/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829260.021700:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829260.021704:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829260.021708:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011762816:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829260.021754:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011762944:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829260.021785:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829260.021800:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829260/real 1787829260] req@ffff907d90844b40 x1874670011762944/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829260.021813:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011762944/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829260.021823:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829260.021827:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829260.021831:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011762944:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829260.148749:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829260.148823:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829261.172717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829261.172834:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829262.196668:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829262.196770:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829263.220686:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829263.220742:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829264.244703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829264.244737:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829265.140719:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829265.140728:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829265.140736:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829265.140744:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4668 00000100:00080000:1.0:1787829265.140753:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829265.140759:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829265.140791:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011763072/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829265.140814:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829265.140816:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829265.140818:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829265.140823:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4668 00000100:00080000:1.0:1787829265.140825:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829265.140828:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829265.140836:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829265.140846:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011763200/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829265.140854:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829265.140856:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829265.140858:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829265.140861:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4668 00000100:00080000:1.0:1787829265.140863:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829265.140866:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829265.140871:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011763072:192.168.203.104@tcp:250:kworker.0 00000008:00100000:1.0:1787829265.140872:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829265.140884:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011763328/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829265.140896:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829265.140899:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829265.140900:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829265.140903:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4668 00000100:00080000:1.0:1787829265.140906:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829265.140908:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829265.140911:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829265.140918:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011763456/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829265.140933:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829265.140968:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829265/real 1787829265] req@ffff907d908a3840 x1874670011763072/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829265.140981:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011763072/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829265.140989:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829265.140991:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829265.140994:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011763072:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829265.140999:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011763200:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829265.141014:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829265.141022:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829265/real 1787829265] req@ffff907d908a21c0 x1874670011763200/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829265.141030:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011763200/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829265.141035:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829265.141037:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829265.141039:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011763200:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829265.141056:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011763328:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829265.141068:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829265.141074:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829265/real 1787829265] req@ffff907d908a2940 x1874670011763328/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829265.141082:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011763328/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829265.141087:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829265.141088:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829265.141090:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011763328:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829265.141094:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011763456:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829265.141103:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829265.141108:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829265/real 1787829265] req@ffff907d908a0000 x1874670011763456/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829265.141115:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011763456/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829265.141125:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829265.141126:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829265.141128:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011763456:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829265.268705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829265.268731:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829266.292763:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829266.292869:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829267.316780:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829267.316869:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829268.340733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829268.340795:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829269.364761:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829269.364805:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829270.260720:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829270.260728:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829270.260732:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829270.260741:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4674 00000100:00080000:1.0:1787829270.260750:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829270.260756:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829270.260787:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011763584/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829270.260810:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829270.260813:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829270.260815:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829270.260820:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4674 00000100:00080000:1.0:1787829270.260823:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829270.260826:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829270.260835:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829270.260839:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011763584:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829270.260849:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011763712/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829270.260860:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829270.260863:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829270.260864:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829270.260868:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4674 00000100:00080000:1.0:1787829270.260871:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829270.260874:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829270.260878:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829270.260888:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011763840/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829270.260894:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829270.260897:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829270.260901:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787829270.260902:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787829270.260907:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4674 00000100:00080000:1.0:1787829270.260913:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829270.260917:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829270.260923:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829270.260937:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011763968/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829270.260956:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829270/real 1787829270] req@ffff907d908a2d00 x1874670011763584/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829270.260977:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011763584/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829270.260996:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829270.261001:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829270.261005:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011763584:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829270.261027:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011763712:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829270.261052:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829270.261066:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829270/real 1787829270] req@ffff907d908a1e00 x1874670011763712/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829270.261081:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011763712/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829270.261091:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829270.261095:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829270.261098:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011763712:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829270.261106:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011763840:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829270.261131:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829270.261143:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829270/real 1787829270] req@ffff907d908a0000 x1874670011763840/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829270.261158:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011763840/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829270.261166:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829270.261170:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829270.261173:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011763840:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829270.261179:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011763968:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829270.261201:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829270.261213:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829270/real 1787829270] req@ffff907d908a2940 x1874670011763968/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829270.261227:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011763968/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829270.261236:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829270.261254:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829270.261259:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011763968:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829270.388749:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829270.388802:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829271.412701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829271.412753:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829272.436727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829272.436745:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829273.460721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829273.460774:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829274.484724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829274.484752:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829275.380700:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829275.380708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829275.380712:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829275.380720:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4679 00000100:00080000:1.0:1787829275.380729:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829275.380734:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829275.380764:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011764096/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829275.380785:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829275.380788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829275.380789:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829275.380792:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4679 00000100:00080000:1.0:1787829275.380794:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829275.380796:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829275.380804:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829275.380813:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011764224/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829275.380821:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829275.380823:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829275.380825:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829275.380829:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011764096:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829275.380830:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4679 00000100:00080000:1.0:1787829275.380835:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829275.380839:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829275.380843:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829275.380850:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011764352/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829275.380867:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829275.380869:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829275.380871:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829275.380873:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4679 00000100:00080000:1.0:1787829275.380876:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829275.380877:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829275.380880:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829275.380887:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011764480/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829275.380901:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829275.380945:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829275/real 1787829275] req@ffff907d908a21c0 x1874670011764096/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829275.380958:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011764096/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829275.380966:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829275.380970:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829275.380972:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011764096:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829275.381000:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011764224:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829275.381015:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829275.381022:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829275/real 1787829275] req@ffff907d908a3840 x1874670011764224/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829275.381030:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011764224/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829275.381035:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829275.381038:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829275.381040:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011764224:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829275.381052:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011764352:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829275.381070:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829275.381076:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829275/real 1787829275] req@ffff907d908a1680 x1874670011764352/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829275.381084:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011764352/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829275.381089:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829275.381091:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829275.381093:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011764352:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829275.381097:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011764480:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829275.381107:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829275.381114:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829275/real 1787829275] req@ffff907d908a2580 x1874670011764480/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829275.381126:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011764480/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829275.381134:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829275.381138:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829275.381142:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011764480:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829275.508725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829275.508768:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829276.532675:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829276.532732:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829277.556711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829277.556772:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829278.580713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829278.580765:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829279.604712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829279.604767:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829280.500722:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829280.500731:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829280.500735:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829280.500743:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4684 00000100:00080000:0.0:1787829280.500753:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829280.500759:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829280.500791:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011764608/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829280.500814:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829280.500817:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829280.500818:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829280.500821:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4684 00000100:00080000:0.0:1787829280.500823:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829280.500827:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829280.500834:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829280.500856:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011764736/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829280.500865:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829280.500870:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011764608:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829280.500871:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829280.500874:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829280.500882:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4684 00000100:00080000:0.0:1787829280.500884:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829280.500886:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829280.500890:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829280.500898:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011764864/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829280.500905:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829280.500917:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829280.500918:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829280.500920:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4684 00000100:00080000:0.0:1787829280.500923:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829280.500925:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829280.500927:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829280.500934:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011764992/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829280.500942:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829280.500979:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829280/real 1787829280] req@ffff907d90847c00 x1874670011764608/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829280.500992:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011764608/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829280.501000:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829280.501002:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829280.501005:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011764608:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829280.501022:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011764736:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829280.501037:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829280.501045:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829280/real 1787829280] req@ffff907d90846580 x1874670011764736/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829280.501053:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011764736/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829280.501058:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829280.501060:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829280.501062:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011764736:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829280.501067:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011764864:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829280.501078:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829280.501083:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829280/real 1787829280] req@ffff907d908470c0 x1874670011764864/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829280.501091:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011764864/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829280.501096:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829280.501097:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829280.501099:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011764864:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829280.501102:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011764992:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829280.501112:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829280.501117:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829280/real 1787829280] req@ffff907d90844b40 x1874670011764992/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829280.501124:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011764992/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829280.501129:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829280.501131:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829280.501132:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011764992:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829280.628705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829280.628854:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829281.652691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829281.652775:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829282.676679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829282.676753:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829283.700690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829283.700728:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829284.724709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829284.724790:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829285.620755:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829285.620769:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829285.620775:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829285.620805:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4689 00000100:00080000:0.0:1787829285.620821:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829285.620832:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829285.620896:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011765120/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829285.620935:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829285.620940:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829285.620942:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829285.620947:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4689 00000100:00080000:0.0:1787829285.620976:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829285.620980:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829285.620990:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829285.621006:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011765248/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829285.621018:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829285.621022:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829285.621024:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829285.621028:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4689 00000100:00080000:0.0:1787829285.621036:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829285.621036:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011765120:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829285.621046:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829285.621053:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829285.621076:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011765376/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829285.621087:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829285.621091:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829285.621093:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829285.621097:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4689 00000100:00080000:0.0:1787829285.621101:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829285.621104:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829285.621109:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787829285.621117:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829285.621123:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011765504/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829285.621167:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829285/real 1787829285] req@ffff907d908452c0 x1874670011765120/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829285.621189:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011765120/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829285.621200:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829285.621205:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829285.621209:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011765120:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829285.621216:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011765248:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829285.621236:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829285.621248:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829285/real 1787829285] req@ffff907d90844000 x1874670011765248/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829285.621259:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011765248/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829285.621267:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829285.621270:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829285.621273:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011765248:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829285.621296:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011765376:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829285.621313:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829285.621323:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829285/real 1787829285] req@ffff907d90845e00 x1874670011765376/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829285.621334:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011765376/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829285.621366:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829285.621371:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829285.621374:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011765376:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829285.621380:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011765504:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829285.621397:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829285.621408:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829285/real 1787829285] req@ffff907d908443c0 x1874670011765504/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829285.621419:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011765504/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829285.621428:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829285.621430:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829285.621433:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011765504:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829285.748727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829285.748811:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829286.772698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829286.772717:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829287.796709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829287.796773:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829288.820667:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829288.820726:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829289.844703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829289.844801:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829290.740705:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829290.740714:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829290.740718:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829290.740728:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4694 00000100:00080000:1.0:1787829290.740741:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829290.740748:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829290.740786:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011765632/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829290.740808:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829290.740811:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829290.740812:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829290.740815:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4694 00000100:00080000:1.0:1787829290.740818:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829290.740821:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829290.740828:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829290.740838:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011765760/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829290.740852:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829290.740857:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1787829290.740857:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011765632:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829290.740860:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829290.740867:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4694 00000100:00080000:1.0:1787829290.740870:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829290.740872:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829290.740875:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829290.740886:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011765888/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829290.740893:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829290.740895:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829290.740897:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829290.740899:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4694 00000100:00080000:1.0:1787829290.740902:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829290.740904:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829290.740907:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829290.740914:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011766016/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829290.740918:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829290.740955:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829290/real 1787829290] req@ffff907d908a12c0 x1874670011765632/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829290.740969:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011765632/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829290.740976:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829290.740979:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829290.740982:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011765632:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829290.740988:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011765760:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829290.741003:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829290.741011:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829290/real 1787829290] req@ffff907d908a0f00 x1874670011765760/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829290.741019:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011765760/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829290.741026:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829290.741030:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829290.741033:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011765760:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829290.741057:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011765888:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829290.741079:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829290.741088:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829290/real 1787829290] req@ffff907d908a2580 x1874670011765888/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829290.741096:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011765888/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829290.741102:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829290.741104:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829290.741106:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011765888:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829290.741110:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011766016:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829290.741122:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829290.741128:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829290/real 1787829290] req@ffff907d908a1680 x1874670011766016/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829290.741135:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011766016/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829290.741140:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829290.741142:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829290.741144:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011766016:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829290.868695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829290.868757:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829291.892684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829291.892758:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829292.916676:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829292.916843:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829293.940713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829293.940732:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829294.964690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829294.964754:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829295.860690:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829295.860700:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829295.860704:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829295.860713:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4699 00000100:00080000:2.0:1787829295.860723:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829295.860729:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829295.860782:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011766144/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829295.860830:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829295.860835:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829295.860838:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829295.860843:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4699 00000100:00080000:2.0:1787829295.860847:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829295.860852:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829295.860866:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829295.860889:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011766272/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829295.860947:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829295.860954:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829295.860958:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829295.860963:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011766144:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829295.860970:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4699 00000100:00080000:2.0:1787829295.860979:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829295.860985:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829295.860992:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829295.861010:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011766400/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829295.861022:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829295.861027:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829295.861029:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829295.861033:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4699 00000100:00080000:2.0:1787829295.861037:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829295.861041:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:0.0:1787829295.861047:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:2.0:1787829295.861051:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829295.861069:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011766528/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829295.861104:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829295/real 1787829295] req@ffff907d911492c0 x1874670011766144/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829295.861125:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011766144/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829295.861137:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829295.861141:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829295.861145:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011766144:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829295.861156:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011766272:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829295.861180:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829295.861193:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829295/real 1787829295] req@ffff907d91148b40 x1874670011766272/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829295.861206:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011766272/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829295.861214:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829295.861217:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829295.861220:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011766272:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829295.861269:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011766400:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829295.861286:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829295.861295:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829295/real 1787829295] req@ffff907d9114a940 x1874670011766400/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829295.861304:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011766400/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829295.861310:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829295.861313:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829295.861315:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011766400:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829295.861319:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011766528:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829295.861330:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829295.861337:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829295/real 1787829295] req@ffff907d91148f00 x1874670011766528/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829295.861345:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011766528/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829295.861351:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829295.861353:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829295.861355:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011766528:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829295.988698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829295.988733:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829297.012715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829297.012808:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829298.036676:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829298.036757:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829299.060701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829299.060746:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829300.084690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829300.084742:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829300.980695:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829300.980703:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829300.980706:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829300.980714:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4704 00000100:00080000:1.0:1787829300.980723:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829300.980729:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829300.980764:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011766656/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829300.980789:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829300.980792:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829300.980794:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829300.980796:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4704 00000100:00080000:1.0:1787829300.980799:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829300.980801:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829300.980809:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829300.980822:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011766784/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829300.980834:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829300.980837:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011766656:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829300.980841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829300.980846:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829300.980851:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4704 00000100:00080000:1.0:1787829300.980856:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829300.980860:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829300.980873:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829300.980884:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011766912/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829300.980891:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829300.980896:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1787829300.980897:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787829300.980898:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829300.980904:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4704 00000100:00080000:1.0:1787829300.980908:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829300.980912:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829300.980915:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829300.980923:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011767040/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829300.980933:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829300/real 1787829300] req@ffff907d908a3840 x1874670011766656/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829300.980949:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011766656/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829300.980957:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829300.980960:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829300.980963:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011766656:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829300.980980:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011766784:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829300.980995:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829300.981007:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829300/real 1787829300] req@ffff907d908a21c0 x1874670011766784/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829300.981020:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011766784/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829300.981034:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829300.981038:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829300.981042:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011766784:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829300.981049:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011766912:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829300.981072:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829300.981084:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829300/real 1787829300] req@ffff907d908a2940 x1874670011766912/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829300.981097:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011766912/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829300.981110:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829300.981113:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829300.981115:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011766912:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829300.981119:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011767040:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829300.981152:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829300.981164:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829300/real 1787829300] req@ffff907d908a0000 x1874670011767040/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829300.981203:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011767040/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829300.981211:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829300.981214:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829300.981219:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011767040:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829301.108705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829301.108739:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829302.132722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829302.132739:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829303.156694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829303.156754:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829304.180725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829304.180799:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829305.204708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829305.204764:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829306.100714:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829306.100720:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829306.100724:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829306.100733:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4709 00000100:00080000:1.0:1787829306.100742:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829306.100748:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829306.100778:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011767168/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829306.100803:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829306.100807:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829306.100808:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829306.100812:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4709 00000100:00080000:1.0:1787829306.100815:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829306.100818:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829306.100826:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829306.100836:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011767168:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829306.100840:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011767296/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829306.100851:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829306.100854:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829306.100855:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829306.100858:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4709 00000100:00080000:1.0:1787829306.100862:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829306.100865:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829306.100869:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829306.100880:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011767424/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829306.100887:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829306.100889:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829306.100890:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829306.100892:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4709 00000100:00080000:1.0:1787829306.100895:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787829306.100897:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787829306.100901:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829306.100906:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829306.100915:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011767552/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829306.100936:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829306/real 1787829306] req@ffff907d908a1e00 x1874670011767168/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829306.100951:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011767168/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829306.100958:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829306.100961:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829306.100964:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011767168:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829306.100983:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011767296:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829306.101078:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829306.101090:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829306/real 1787829306] req@ffff907d908a2d00 x1874670011767296/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829306.101099:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011767296/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829306.101106:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829306.101110:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829306.101113:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011767296:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829306.101120:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011767424:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829306.101244:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829306.101256:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829306/real 1787829306] req@ffff907d908a0000 x1874670011767424/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829306.101266:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011767424/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829306.101273:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829306.101276:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829306.101279:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011767424:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829306.101335:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011767552:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829306.101352:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829306.101360:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829306/real 1787829306] req@ffff907d908a2940 x1874670011767552/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829306.101368:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011767552/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829306.101374:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829306.101376:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829306.101378:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011767552:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829306.228693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829306.228757:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829307.252714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829307.252828:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829308.276717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829308.276746:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829309.300707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829309.300762:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829310.324698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829310.324755:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829311.220707:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829311.220723:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829311.220728:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829311.220742:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4714 00000100:00080000:3.0:1787829311.220756:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829311.220765:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829311.220814:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011767680/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829311.220848:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829311.220853:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829311.220855:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829311.220860:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4714 00000100:00080000:3.0:1787829311.220865:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829311.220871:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829311.220883:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829311.220898:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011767680:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829311.220905:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011767808/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829311.220919:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829311.220924:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829311.220926:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829311.220931:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4714 00000100:00080000:3.0:1787829311.220936:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829311.220941:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829311.220946:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829311.220960:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011767936/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829311.220972:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829311.220976:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829311.220978:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829311.220983:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4714 00000100:00080000:3.0:1787829311.220987:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829311.220993:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829311.220999:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829311.221012:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011767808:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829311.221015:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011768064/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829311.221046:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011767936:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829311.221068:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011768064:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829311.348717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829311.348780:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829312.372720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829312.372778:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829313.396694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829313.396759:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829314.292833:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787829314.292871:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829314.292885:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829314.292930:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829314.292943:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829314.292954:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:0.0:1787829314.292964:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829311/real 1787829314] req@ffff907da562da40 x1874670011767808/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829314.292986:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011767808/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829314.292996:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829314.292999:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829314.293002:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011767808:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829314.293008:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829311/real 1787829314] req@ffff907da562cb40 x1874670011767936/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829314.293015:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011767936/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829314.293021:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829314.293024:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829314.293028:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011767936:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829314.293036:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829311/real 1787829314] req@ffff907da562de00 x1874670011767680/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829314.293048:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011767680/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829314.293055:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829314.293059:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829314.293063:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011767680:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829314.293086:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829311/real 1787829314] req@ffff907da562e1c0 x1874670011768064/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829314.293099:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011768064/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829314.293108:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829314.293111:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829314.293115:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011768064:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829314.420688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829314.420744:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829315.444682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829315.444711:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829316.340710:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829316.340718:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829316.340722:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829316.340732:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4720 00000100:00080000:0.0:1787829316.340741:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829316.340746:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829316.340779:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011768192/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829316.340802:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829316.340805:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829316.340807:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829316.340810:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4720 00000100:00080000:0.0:1787829316.340812:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829316.340815:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829316.340823:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829316.340835:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011768320/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829316.340847:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829316.340853:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1787829316.340854:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011768192:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829316.340856:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829316.340866:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4720 00000100:00080000:0.0:1787829316.340877:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829316.340882:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829316.340888:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829316.340901:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011768448/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829316.340911:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829316.340915:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829316.340917:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1787829316.340922:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787829316.340923:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4720 00000100:00080000:0.0:1787829316.340931:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829316.340936:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829316.340942:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829316.340955:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011768576/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829316.340967:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829316/real 1787829316] req@ffff907d90844780 x1874670011768192/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829316.340990:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011768192/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829316.341002:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829316.341007:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829316.341011:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011768192:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829316.341036:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011768320:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829316.341062:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829316.341077:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829316/real 1787829316] req@ffff907d90845680 x1874670011768320/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829316.341086:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011768320/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829316.341092:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829316.341094:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829316.341097:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011768320:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829316.341101:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011768448:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829316.341128:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829316.341141:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829316/real 1787829316] req@ffff907d90846d00 x1874670011768448/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829316.341155:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011768448/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829316.341170:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829316.341174:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829316.341177:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011768448:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829316.341183:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011768576:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829316.341204:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829316.341217:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829316/real 1787829316] req@ffff907d90845a40 x1874670011768576/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829316.341230:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011768576/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829316.341238:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829316.341241:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829316.341245:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011768576:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787829316.468753:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829316.468797:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787829317.492687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829317.492754:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829318.516675:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829318.516739:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829319.540668:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829319.540752:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829320.564699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829320.564800:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829321.460712:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829321.460721:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829321.460725:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829321.460733:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4725 00000100:00080000:0.0:1787829321.460743:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829321.460748:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829321.460780:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011768704/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829321.460813:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829321.460816:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829321.460817:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829321.460820:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4725 00000100:00080000:0.0:1787829321.460823:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829321.460825:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829321.460832:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829321.460841:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011768832/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829321.460848:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829321.460852:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829321.460854:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829321.460857:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4725 00000100:00080000:0.0:1787829321.460859:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829321.460861:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829321.460864:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829321.460869:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011768704:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829321.460874:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011768960/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829321.460883:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829321.460885:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829321.460886:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829321.460889:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4725 00000100:00080000:0.0:1787829321.460891:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829321.460893:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829321.460895:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829321.460904:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011769088/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829321.460937:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829321.460978:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829321/real 1787829321] req@ffff907d90847480 x1874670011768704/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829321.460996:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011768704/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829321.461007:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829321.461011:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829321.461015:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011768704:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829321.461023:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011768832:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829321.461043:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829321.461053:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829321/real 1787829321] req@ffff907d90846940 x1874670011768832/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829321.461062:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011768832/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829321.461068:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829321.461070:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829321.461072:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011768832:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829321.461089:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011768960:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829321.461101:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829321.461107:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829321/real 1787829321] req@ffff907d90844f00 x1874670011768960/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829321.461115:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011768960/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829321.461120:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829321.461122:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829321.461124:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011768960:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829321.461128:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011769088:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829321.461136:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829321.461142:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829321/real 1787829321] req@ffff907d90845a40 x1874670011769088/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829321.461149:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011769088/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829321.461159:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829321.461161:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829321.461163:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011769088:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829321.588723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829321.588776:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829322.612681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829322.612706:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829323.636685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829323.636783:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829324.660672:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829324.660752:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829325.684674:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829325.684739:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829326.580751:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829326.580759:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829326.580763:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829326.580772:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4730 00000100:00080000:1.0:1787829326.580781:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829326.580787:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829326.580825:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011769216/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829326.580847:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829326.580851:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829326.580853:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829326.580857:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4730 00000100:00080000:1.0:1787829326.580861:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829326.580865:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829326.580875:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829326.580889:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011769344/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829326.580891:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011769216:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829326.580900:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829326.580904:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829326.580906:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829326.580910:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4730 00000100:00080000:1.0:1787829326.580913:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829326.580917:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829326.580922:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829326.580931:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011769472/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829326.580939:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829326.580942:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829326.580944:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829326.580947:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4730 00000100:00080000:1.0:1787829326.580950:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829326.580953:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829326.580957:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787829326.580962:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829326.580965:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011769600/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829326.580998:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829326/real 1787829326] req@ffff907d908a21c0 x1874670011769216/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829326.581017:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011769216/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829326.581026:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829326.581029:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829326.581032:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011769216:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829326.581051:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011769344:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829326.581076:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829326.581090:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829326/real 1787829326] req@ffff907d908a3840 x1874670011769344/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829326.581104:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011769344/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829326.581111:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829326.581113:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829326.581116:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011769344:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829326.581120:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011769472:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829326.581136:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829326.581143:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829326/real 1787829326] req@ffff907d908a1680 x1874670011769472/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829326.581151:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011769472/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829326.581167:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829326.581168:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829326.581170:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011769472:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829326.581174:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011769600:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829326.581183:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829326.581189:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829326/real 1787829326] req@ffff907d908a2580 x1874670011769600/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829326.581196:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011769600/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829326.581201:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829326.581203:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829326.581205:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011769600:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829326.708708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829326.708772:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829327.732672:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829327.732734:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829328.756721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829328.756801:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829329.780748:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829329.780776:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829330.804696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829330.804761:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829331.700710:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829331.700718:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829331.700722:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829331.700730:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4735 00000100:00080000:0.0:1787829331.700739:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829331.700745:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829331.700777:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011769728/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829331.700798:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829331.700801:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829331.700803:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829331.700805:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4735 00000100:00080000:0.0:1787829331.700808:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829331.700810:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829331.700826:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829331.700835:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011769856/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829331.700844:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829331.700848:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829331.700850:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787829331.700850:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011769728:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829331.700857:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4735 00000100:00080000:0.0:1787829331.700862:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829331.700865:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829331.700868:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829331.700878:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011769984/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829331.700885:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829331.700893:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829331.700895:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829331.700898:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4735 00000100:00080000:0.0:1787829331.700901:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829331.700903:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829331.700906:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829331.700914:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011770112/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829331.700924:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829331.700974:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829331/real 1787829331] req@ffff907d90846d00 x1874670011769728/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829331.700988:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011769728/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829331.700995:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829331.700998:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829331.701001:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011769728:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829331.701019:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011769856:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829331.701036:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829331.701044:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829331/real 1787829331] req@ffff907d90845680 x1874670011769856/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829331.701053:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011769856/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829331.701058:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829331.701061:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829331.701064:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011769856:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829331.701068:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011769984:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829331.701079:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829331.701085:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829331/real 1787829331] req@ffff907d90844780 x1874670011769984/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829331.701093:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011769984/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829331.701098:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829331.701100:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829331.701102:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011769984:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829331.701105:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011770112:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829331.701114:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829331.701120:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829331/real 1787829331] req@ffff907d908443c0 x1874670011770112/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829331.701127:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011770112/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829331.701132:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829331.701134:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829331.701136:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011770112:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829331.828720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829331.828755:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829332.852709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829332.852767:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829333.876689:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829333.876749:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829334.900703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829334.900762:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829335.924684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829335.924771:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829336.820790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829336.820801:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829336.820806:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829336.820817:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4740 00000100:00080000:1.0:1787829336.820830:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829336.820838:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829336.820885:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011770240/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829336.820908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829336.820911:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829336.820912:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829336.820915:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4740 00000100:00080000:1.0:1787829336.820918:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829336.820920:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829336.820930:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829336.820940:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011770368/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829336.820947:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829336.820951:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829336.820952:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829336.820956:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4740 00000100:00080000:1.0:1787829336.820959:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787829336.820962:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011770240:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829336.820964:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829336.820972:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829336.820982:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011770496/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829336.820989:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829336.820992:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829336.820993:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829336.820996:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4740 00000100:00080000:1.0:1787829336.820998:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829336.821000:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829336.821003:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829336.821011:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011770624/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829336.821035:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829336.821077:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829336/real 1787829336] req@ffff907d908a0f00 x1874670011770240/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829336.821092:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011770240/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829336.821102:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829336.821105:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829336.821108:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011770240:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829336.821113:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011770368:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829336.821128:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829336.821137:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829336/real 1787829336] req@ffff907d908a12c0 x1874670011770368/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829336.821145:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011770368/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829336.821151:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829336.821153:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829336.821156:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011770368:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829336.821194:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011770496:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829336.821214:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829336.821222:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829336/real 1787829336] req@ffff907d908a2580 x1874670011770496/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829336.821230:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011770496/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829336.821236:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829336.821239:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829336.821241:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011770496:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829336.821245:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011770624:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829336.821255:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829336.821260:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829336/real 1787829336] req@ffff907d908a1680 x1874670011770624/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829336.821268:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011770624/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829336.821280:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829336.821282:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829336.821284:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011770624:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829336.948736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829336.948790:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829337.972687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829337.972751:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829338.996745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829338.996806:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829340.020689:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829340.020733:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829341.044688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829341.044753:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829341.940747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829341.940754:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829341.940757:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829341.940766:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4745 00000100:00080000:3.0:1787829341.940783:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829341.940791:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829341.940821:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011770752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829341.940842:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829341.940845:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829341.940846:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829341.940849:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4745 00000100:00080000:3.0:1787829341.940851:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829341.940854:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829341.940862:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829341.940875:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011770880/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829341.940883:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829341.940886:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829341.940889:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829341.940891:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011770752:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829341.940895:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4745 00000100:00080000:3.0:1787829341.940901:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829341.940904:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829341.940909:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829341.940919:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011771008/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829341.940926:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829341.940929:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829341.940930:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829341.940933:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4745 00000100:00080000:3.0:1787829341.940936:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829341.940939:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829341.940942:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829341.940950:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011771136/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829341.940966:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829341.941008:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829341/real 1787829341] req@ffff907da562f840 x1874670011770752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829341.941027:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011770752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829341.941039:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829341.941043:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829341.941047:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011770752:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829341.941071:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011770880:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829341.941096:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829341.941110:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829341/real 1787829341] req@ffff907da562f0c0 x1874670011770880/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829341.941132:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011770880/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829341.941156:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829341.941160:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829341.941164:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011770880:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829341.941170:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011771008:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829341.941190:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829341.941202:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829341/real 1787829341] req@ffff907da562c3c0 x1874670011771008/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829341.941215:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011771008/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829341.941223:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829341.941226:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829341.941229:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011771008:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829341.941236:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011771136:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829341.941255:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829341.941267:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829341/real 1787829341] req@ffff907da562e940 x1874670011771136/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829341.941279:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011771136/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829341.941295:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829341.941298:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829341.941301:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011771136:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829342.068697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829342.068701:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829343.092681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829343.092744:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829344.116682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829344.116733:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829345.140680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829345.140750:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829346.164705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829346.164754:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829347.060709:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829347.060719:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829347.060722:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829347.060731:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4750 00000100:00080000:3.0:1787829347.060742:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829347.060749:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829347.060788:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011771264/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829347.060818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829347.060821:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829347.060822:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829347.060825:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4750 00000100:00080000:3.0:1787829347.060828:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829347.060831:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829347.060839:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829347.060849:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011771392/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829347.060857:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829347.060859:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829347.060861:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829347.060864:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4750 00000100:00080000:3.0:1787829347.060866:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829347.060868:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829347.060872:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829347.060881:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011771520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829347.060882:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011771264:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829347.060893:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829347.060895:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829347.060897:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829347.060898:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4750 00000100:00080000:3.0:1787829347.060901:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829347.060903:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829347.060906:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829347.060919:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011771648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829347.060954:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829347.060995:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829347/real 1787829347] req@ffff907da562c000 x1874670011771264/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829347.061013:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011771264/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829347.061025:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829347.061029:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829347.061034:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011771264:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829347.061042:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011771392:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829347.061067:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829347.061082:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829347/real 1787829347] req@ffff907da562fc00 x1874670011771392/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829347.061096:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011771392/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829347.061121:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829347.061126:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829347.061130:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011771392:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829347.061155:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011771520:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829347.061180:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829347.061194:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829347/real 1787829347] req@ffff907da562d2c0 x1874670011771520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829347.061222:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011771520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829347.061232:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829347.061236:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829347.061240:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011771520:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829347.061246:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011771648:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829347.061268:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829347.061280:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829347/real 1787829347] req@ffff907da562e580 x1874670011771648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829347.061294:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011771648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829347.061303:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829347.061307:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829347.061311:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011771648:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829347.188667:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829347.188711:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829348.212696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829348.212775:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829349.236707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829349.236740:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829350.260714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829350.260735:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829351.284713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829351.284764:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829352.180705:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829352.180713:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829352.180717:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829352.180725:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4755 00000100:00080000:0.0:1787829352.180734:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829352.180740:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829352.180769:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011771776/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829352.180789:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829352.180793:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829352.180795:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829352.180799:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4755 00000100:00080000:0.0:1787829352.180803:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829352.180807:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829352.180817:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829352.180829:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011771904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829352.180829:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011771776:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829352.180839:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829352.180842:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829352.180844:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829352.180848:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4755 00000100:00080000:0.0:1787829352.180851:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829352.180854:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829352.180859:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829352.180868:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011772032/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829352.180877:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829352.180880:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829352.180882:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829352.180885:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4755 00000100:00080000:0.0:1787829352.180887:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829352.180891:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:3.0:1787829352.180894:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:0.0:1787829352.180896:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829352.180905:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011772160/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829352.180937:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829352/real 1787829352] req@ffff907d90845e00 x1874670011771776/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829352.180954:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011771776/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829352.180962:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829352.180964:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829352.180967:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011771776:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829352.180986:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011771904:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829352.181003:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829352.181012:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829352/real 1787829352] req@ffff907d90844000 x1874670011771904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829352.181024:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011771904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829352.181029:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829352.181031:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829352.181033:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011771904:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829352.181036:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011772032:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829352.181048:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829352.181054:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829352/real 1787829352] req@ffff907d908452c0 x1874670011772032/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829352.181062:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011772032/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829352.181082:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829352.181085:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829352.181087:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011772032:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829352.181091:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011772160:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829352.181105:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829352.181112:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829352/real 1787829352] req@ffff907d90844b40 x1874670011772160/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829352.181119:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011772160/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829352.181129:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829352.181131:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829352.181133:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011772160:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829352.308750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829352.308798:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829353.332680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829353.332722:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829354.356704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829354.356735:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829355.380680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829355.380748:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829356.404709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829356.404801:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829357.300766:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829357.300774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829357.300779:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829357.300790:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4761 00000100:00080000:1.0:1787829357.300799:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829357.300805:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829357.300846:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011772288/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829357.300865:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829357.300868:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829357.300869:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829357.300872:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4761 00000100:00080000:1.0:1787829357.300874:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829357.300877:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829357.300886:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829357.300895:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011772416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829357.300903:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829357.300907:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829357.300909:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787829357.300911:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011772288:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829357.300913:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4761 00000100:00080000:1.0:1787829357.300918:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829357.300923:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829357.300926:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829357.300934:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011772544/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829357.300945:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829357.300948:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829357.300949:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829357.300952:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4761 00000100:00080000:1.0:1787829357.300954:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829357.300957:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829357.300959:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829357.300968:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011772672/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829357.300977:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829357.301013:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829357/real 1787829357] req@ffff907d908a3840 x1874670011772288/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829357.301026:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011772288/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829357.301050:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829357.301054:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829357.301057:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011772288:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829357.301075:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011772416:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829357.301094:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829357.301103:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829357/real 1787829357] req@ffff907d908a21c0 x1874670011772416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829357.301112:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011772416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829357.301118:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829357.301120:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829357.301122:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011772416:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829357.301126:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011772544:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829357.301138:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829357.301144:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829357/real 1787829357] req@ffff907d908a2940 x1874670011772544/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829357.301152:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011772544/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829357.301157:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829357.301158:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829357.301160:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011772544:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829357.301164:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011772672:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829357.301174:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829357.301179:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829357/real 1787829357] req@ffff907d908a0000 x1874670011772672/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829357.301187:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011772672/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829357.301191:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829357.301193:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829357.301196:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011772672:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829357.428702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829357.428846:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829358.452742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829358.452829:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829359.476694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829359.476794:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829360.500696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829360.500729:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829361.524775:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829361.524861:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:1.0:1787829362.420711:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829362.420719:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829362.420723:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829362.420732:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4766 00000100:00080000:1.0:1787829362.420741:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829362.420747:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829362.420779:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011772800/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829362.420802:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829362.420805:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829362.420806:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829362.420809:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4766 00000100:00080000:1.0:1787829362.420811:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829362.420814:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829362.420822:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829362.420830:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011772928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829362.420838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829362.420841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829362.420843:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829362.420846:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4766 00000100:00080000:1.0:1787829362.420848:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829362.420850:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829362.420853:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829362.420860:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011773056/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829362.420868:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829362.420871:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829362.420874:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829362.420881:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4766 00000100:00100000:3.0:1787829362.420881:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011772800:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829362.420892:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829362.420900:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829362.420903:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829362.420918:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011773184/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829362.420973:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829362.421035:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829362/real 1787829362] req@ffff907d908a2d00 x1874670011772800/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829362.421052:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011772800/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829362.421062:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829362.421065:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829362.421069:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011772800:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829362.421076:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011772928:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829362.421096:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829362.421104:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829362/real 1787829362] req@ffff907d908a1e00 x1874670011772928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829362.421113:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011772928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829362.421119:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829362.421122:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829362.421124:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011772928:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829362.421151:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011773056:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829362.421164:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829362.421171:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829362/real 1787829362] req@ffff907d908a0000 x1874670011773056/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829362.421179:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011773056/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829362.421185:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829362.421187:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829362.421189:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011773056:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829362.421194:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011773184:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829362.421210:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829362.421222:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829362/real 1787829362] req@ffff907d908a2940 x1874670011773184/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829362.421235:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011773184/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829362.421245:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829362.421249:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829362.421252:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011773184:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829362.548730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829362.548757:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829363.572678:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829363.572705:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829364.596692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829364.596737:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829365.620690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829365.620760:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829366.644661:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829366.644729:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829367.540674:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829367.540683:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829367.540686:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829367.540695:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4771 00000100:00080000:1.0:1787829367.540705:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829367.540710:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829367.540742:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011773312/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829367.540764:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829367.540766:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829367.540768:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829367.540774:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4771 00000100:00080000:1.0:1787829367.540777:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829367.540779:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829367.540787:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829367.540796:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011773440/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829367.540804:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829367.540807:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829367.540808:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829367.540810:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4771 00000100:00080000:1.0:1787829367.540813:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787829367.540817:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011773312:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829367.540818:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829367.540824:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829367.540832:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011773568/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829367.540839:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829367.540841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829367.540843:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829367.540845:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4771 00000100:00080000:1.0:1787829367.540846:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829367.540848:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829367.540851:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829367.540861:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011773696/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829367.540890:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829367.540934:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829367/real 1787829367] req@ffff907d908a21c0 x1874670011773312/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829367.540953:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011773312/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829367.540979:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829367.540984:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829367.540988:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011773312:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829367.540996:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011773440:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829367.541029:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829367.541044:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829367/real 1787829367] req@ffff907d908a3840 x1874670011773440/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829367.541058:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011773440/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829367.541067:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829367.541070:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829367.541074:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011773440:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829367.541097:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011773568:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829367.541119:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829367.541131:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829367/real 1787829367] req@ffff907d908a1680 x1874670011773568/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829367.541140:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011773568/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829367.541150:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829367.541152:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829367.541154:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011773568:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829367.541158:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011773696:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829367.541170:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829367.541176:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829367/real 1787829367] req@ffff907d908a2580 x1874670011773696/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829367.541184:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011773696/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829367.541189:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829367.541190:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829367.541192:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011773696:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829367.668700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829367.668776:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829368.692688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829368.692710:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829369.716716:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829369.716771:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787829370.740708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829370.740740:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829371.764736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829371.764801:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829372.660756:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829372.660768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829372.660774:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829372.660788:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4776 00000100:00080000:2.0:1787829372.660803:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829372.660824:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829372.660873:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011773824/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829372.660905:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829372.660912:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829372.660915:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829372.660920:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4776 00000100:00080000:2.0:1787829372.660924:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829372.660944:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829372.660961:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829372.660965:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011773824:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829372.660978:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011773952/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829372.660992:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829372.660997:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829372.660999:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829372.661005:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4776 00000100:00080000:2.0:1787829372.661025:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829372.661030:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:3.0:1787829372.661035:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:2.0:1787829372.661037:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829372.661050:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011774080/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829372.661058:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829372.661061:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829372.661063:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829372.661065:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4776 00000100:00080000:2.0:1787829372.661069:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829372.661071:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829372.661076:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829372.661086:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829372/real 1787829372] req@ffff907d91148f00 x1874670011773824/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829372.661087:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011774208/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829372.661124:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011773824/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829372.661138:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829372.661144:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829372.661147:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011773824:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829372.661170:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011773952:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829372.661190:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829372.661200:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829372/real 1787829372] req@ffff907d9114a940 x1874670011773952/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829372.661209:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011773952/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829372.661215:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829372.661217:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829372.661221:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011773952:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829372.661227:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011774080:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829372.661247:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829372.661260:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829372/real 1787829372] req@ffff907d91148b40 x1874670011774080/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829372.661272:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011774080/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829372.661279:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829372.661281:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829372.661284:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011774080:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829372.661288:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011774208:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829372.661303:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829372.661311:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829372/real 1787829372] req@ffff907d911492c0 x1874670011774208/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829372.661318:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011774208/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829372.661324:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829372.661326:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829372.661327:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011774208:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829372.788761:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829372.788813:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829373.812731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829373.812757:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829374.836737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829374.836799:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829375.860729:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829375.860737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787829376.884740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829376.884762:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829377.780725:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829377.780735:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829377.780739:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829377.780748:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4781 00000100:00080000:3.0:1787829377.780758:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829377.780763:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829377.780797:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011774336/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829377.780823:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829377.780827:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829377.780830:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829377.780833:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4781 00000100:00080000:3.0:1787829377.780838:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829377.780843:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829377.780853:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829377.780865:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011774464/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829377.780878:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011774336:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829377.780878:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829377.780908:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829377.780913:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829377.780920:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4781 00000100:00080000:3.0:1787829377.780924:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829377.780927:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829377.780936:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829377.780952:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011774592/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829377.780963:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829377.780967:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829377.780970:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829377.780980:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4781 00000100:00080000:3.0:1787829377.780986:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829377.780992:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829377.781039:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011774464:192.168.203.104@tcp:38:kworker.0 00000008:00100000:3.0:1787829377.781046:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829377.781062:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011774720/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829377.781079:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011774592:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829377.781107:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011774720:192.168.203.104@tcp:8:kworker.0 00010000:00010000:0.0:1787829377.908753:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829377.908769:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787829378.932762:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829378.932774:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829379.956706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829379.956759:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787829380.852865:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:0.0:1787829380.852930:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787829380.852950:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787829380.852992:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787829380.853005:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787829380.853015:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:2.0:1787829380.853043:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829377/real 1787829380] req@ffff907da562cf00 x1874670011774720/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829380.853070:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011774720/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829380.853082:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829380.853088:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829380.853092:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011774720:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829380.853101:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829377/real 1787829380] req@ffff907da562c780 x1874670011774464/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829380.853116:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011774464/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829380.853126:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829380.853130:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829380.853134:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011774464:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829380.853141:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829377/real 1787829380] req@ffff907da562ed00 x1874670011774592/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829380.853155:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011774592/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829380.853164:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829380.853168:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829380.853171:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011774592:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829380.853178:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829377/real 1787829380] req@ffff907da562f480 x1874670011774336/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829380.853191:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011774336/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829380.853199:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829380.853203:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829380.853206:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011774336:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829380.980682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829380.980742:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829382.004684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829382.004749:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829382.900709:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829382.900718:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829382.900723:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829382.900734:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4786 00000100:00080000:0.0:1787829382.900746:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829382.900754:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829382.900788:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011774848/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829382.900819:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829382.900824:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829382.900826:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829382.900830:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4786 00000100:00080000:0.0:1787829382.900834:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829382.900838:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829382.900848:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829382.900876:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011774976/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829382.900885:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011774848:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829382.900892:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829382.900897:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829382.900899:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829382.900904:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4786 00000100:00080000:0.0:1787829382.900909:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829382.900914:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829382.900919:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829382.900932:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011775104/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829382.900943:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829382.900946:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829382.900948:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829382.900952:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4786 00000100:00080000:0.0:1787829382.900957:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787829382.900958:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787829382.900964:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829382.900970:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829382.900983:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011775232/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829382.901006:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829382/real 1787829382] req@ffff907d908470c0 x1874670011774848/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829382.901030:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011774848/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829382.901041:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829382.901046:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829382.901050:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011774848:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829382.901083:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011774976:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829382.901109:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829382.901124:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829382/real 1787829382] req@ffff907d90846580 x1874670011774976/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829382.901137:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011774976/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829382.901145:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829382.901148:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829382.901152:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011774976:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829382.901158:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011775104:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829382.901179:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829382.901190:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829382/real 1787829382] req@ffff907d90847c00 x1874670011775104/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829382.901202:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011775104/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829382.901216:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829382.901220:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829382.901223:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011775104:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829382.901229:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011775232:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829382.901249:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829382.901262:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829382/real 1787829382] req@ffff907d90844b40 x1874670011775232/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829382.901274:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011775232/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829382.901283:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829382.901287:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829382.901290:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011775232:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829383.028682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829383.028745:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829384.052686:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829384.052776:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829385.076701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829385.076721:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829386.100693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829386.100716:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829387.124699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829387.124767:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829388.020704:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829388.020714:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829388.020718:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829388.020728:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4791 00000100:00080000:3.0:1787829388.020739:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829388.020746:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829388.020781:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011775360/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829388.020807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829388.020812:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829388.020814:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829388.020819:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4791 00000100:00080000:3.0:1787829388.020837:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829388.020842:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829388.020852:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829388.020857:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011775360:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829388.020872:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011775488/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829388.020894:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829388.020898:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829388.020901:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829388.020906:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4791 00000100:00080000:3.0:1787829388.020911:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787829388.020912:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787829388.020920:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829388.020927:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829388.020941:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011775616/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829388.020952:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829388.020954:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829388/real 1787829388] req@ffff907da562d680 x1874670011775360/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829388.020959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829388.020964:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829388.020972:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4791 00000100:00080000:3.0:1787829388.020976:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829388.020978:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011775360/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829388.020984:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00080000:2.0:1787829388.020989:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1787829388.020991:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1787829388.020994:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829388.020999:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011775360:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829388.021013:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011775744/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829388.021020:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011775488:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829388.021041:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829388.021053:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829388/real 1787829388] req@ffff907da562f480 x1874670011775488/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829388.021061:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011775488/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829388.021066:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829388.021068:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829388.021070:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011775488:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829388.021074:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011775616:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829388.021085:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829388.021091:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829388/real 1787829388] req@ffff907da562ed00 x1874670011775616/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829388.021098:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011775616/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829388.021104:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829388.021105:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829388.021107:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011775616:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829388.021116:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011775744:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829388.021126:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829388.021132:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829388/real 1787829388] req@ffff907da562c780 x1874670011775744/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829388.021139:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011775744/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829388.021144:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829388.021146:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829388.021148:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011775744:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829388.148683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829388.148742:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829389.172706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829389.172762:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829390.196699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829390.196765:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829391.220717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829391.220746:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829392.244723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829392.244752:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829393.140724:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829393.140735:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829393.140740:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829393.140749:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4796 00000100:00080000:0.0:1787829393.140761:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829393.140770:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829393.140819:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011775872/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829393.140846:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829393.140850:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829393.140852:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829393.140856:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4796 00000100:00080000:0.0:1787829393.140860:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829393.140866:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829393.140876:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829393.140886:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011775872:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829393.140894:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011776000/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829393.140907:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829393.140911:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829393.140913:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829393.140917:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4796 00000100:00080000:0.0:1787829393.140921:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829393.140926:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829393.140932:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829393.140947:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011776128/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829393.140952:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787829393.140965:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829393.140968:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829393.140970:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829393.140974:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4796 00000100:00080000:0.0:1787829393.140977:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829393.140981:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829393.140987:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829393.141001:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011776256/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829393.141008:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829393/real 1787829393] req@ffff907d908452c0 x1874670011775872/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829393.141030:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011775872/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829393.141041:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829393.141046:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829393.141051:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011775872:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829393.141072:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011776000:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829393.141092:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829393.141104:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829393/real 1787829393] req@ffff907d90844000 x1874670011776000/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829393.141116:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011776000/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829393.141133:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829393.141136:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829393.141140:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011776000:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829393.141146:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011776128:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829393.141162:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829393.141171:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829393/real 1787829393] req@ffff907d90845e00 x1874670011776128/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829393.141191:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011776128/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829393.141200:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829393.141203:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829393.141206:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011776128:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829393.141212:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011776256:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829393.141228:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829393.141237:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829393/real 1787829393] req@ffff907d908443c0 x1874670011776256/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829393.141250:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011776256/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829393.141258:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829393.141261:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829393.141265:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011776256:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829393.268706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829393.268728:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829394.292713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829394.292775:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829395.316746:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829395.316779:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787829396.340695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829396.340756:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829397.364698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829397.364751:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829398.260706:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829398.260715:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829398.260719:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829398.260729:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4802 00000100:00080000:1.0:1787829398.260739:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829398.260762:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829398.260801:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011776384/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829398.260821:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829398.260824:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829398.260826:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829398.260829:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4802 00000100:00080000:1.0:1787829398.260831:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829398.260834:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829398.260842:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829398.260852:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011776512/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829398.260860:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829398.260863:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829398.260864:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829398.260867:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4802 00000100:00080000:1.0:1787829398.260869:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829398.260872:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011776384:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829398.260874:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829398.260883:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829398.260892:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011776640/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829398.260904:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829398.260907:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829398.260908:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829398.260911:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4802 00000100:00080000:1.0:1787829398.260913:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829398.260916:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829398.260919:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829398.260926:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011776768/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829398.260934:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829398.260970:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829398/real 1787829398] req@ffff907d908a12c0 x1874670011776384/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829398.260984:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011776384/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829398.260991:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829398.260994:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829398.260997:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011776384:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829398.261002:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011776512:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829398.261026:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829398.261034:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829398/real 1787829398] req@ffff907d908a0f00 x1874670011776512/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829398.261042:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011776512/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829398.261047:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829398.261049:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829398.261052:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011776512:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829398.261069:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011776640:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829398.261081:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829398.261089:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829398/real 1787829398] req@ffff907d908a2580 x1874670011776640/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829398.261096:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011776640/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829398.261101:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829398.261103:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829398.261105:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011776640:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829398.261108:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011776768:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829398.261118:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829398.261124:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829398/real 1787829398] req@ffff907d908a1680 x1874670011776768/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829398.261131:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011776768/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829398.261136:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829398.261137:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829398.261139:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011776768:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829398.388735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829398.388783:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829399.412710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829399.412760:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829400.436746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829400.436786:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829401.460691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829401.460829:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829402.484742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829402.484772:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829403.380800:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829403.380811:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829403.380815:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829403.380824:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4807 00000100:00080000:0.0:1787829403.380833:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829403.380838:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829403.380872:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011776896/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829403.380904:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829403.380908:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829403.380910:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829403.380915:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4807 00000100:00080000:0.0:1787829403.380920:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829403.380925:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829403.380936:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829403.380950:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011777024/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829403.380960:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829403.380967:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1787829403.380969:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011776896:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829403.380970:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829403.380982:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4807 00000100:00080000:0.0:1787829403.380987:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829403.380991:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829403.380998:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829403.381013:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011777152/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829403.381024:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829403.381027:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829403.381028:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829403.381031:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4807 00000100:00080000:0.0:1787829403.381035:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829403.381037:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829403.381042:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829403.381051:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011777280/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829403.381057:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829403.381118:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829403/real 1787829403] req@ffff907d90844780 x1874670011776896/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829403.381134:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011776896/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829403.381144:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829403.381147:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829403.381151:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011776896:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829403.381175:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011777024:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829403.381192:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829403.381201:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829403/real 1787829403] req@ffff907d90845680 x1874670011777024/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829403.381210:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011777024/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829403.381215:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829403.381218:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829403.381220:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011777024:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829403.381224:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011777152:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829403.381236:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829403.381246:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829403/real 1787829403] req@ffff907d90846d00 x1874670011777152/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829403.381260:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011777152/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829403.381269:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829403.381272:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829403.381276:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011777152:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829403.381284:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011777280:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829403.381306:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829403.381319:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829403/real 1787829403] req@ffff907d90845a40 x1874670011777280/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829403.381333:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011777280/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829403.381341:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829403.381345:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829403.381349:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011777280:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829403.508669:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829403.508760:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829404.532742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829404.532815:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829405.556738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829405.556789:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829406.580717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829406.580744:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829407.604695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829407.604766:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829408.500744:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829408.500752:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829408.500755:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829408.500765:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4812 00000100:00080000:0.0:1787829408.500776:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829408.500784:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829408.500823:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011777408/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829408.500848:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829408.500851:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829408.500852:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829408.500856:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4812 00000100:00080000:0.0:1787829408.500858:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829408.500861:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829408.500869:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829408.500879:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011777536/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829408.500886:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829408.500889:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829408.500890:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829408.500893:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4812 00000100:00080000:0.0:1787829408.500895:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829408.500898:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829408.500902:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829408.500910:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011777664/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829408.500916:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829408.500918:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829408.500919:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829408.500922:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4812 00000100:00080000:0.0:1787829408.500925:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829408.500928:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829408.500931:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829408.500945:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011777792/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829408.500952:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011777408:192.168.203.104@tcp:250:kworker.0 00000800:00000100:2.0:1787829408.501044:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829408.501106:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829408/real 1787829408] req@ffff907d90844f00 x1874670011777408/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829408.501130:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011777408/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829408.501144:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829408.501149:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829408.501154:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011777408:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829408.501163:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011777536:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829408.501190:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829408.501205:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829408/real 1787829408] req@ffff907d90846940 x1874670011777536/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829408.501219:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011777536/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829408.501225:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829408.501228:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829408.501230:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011777536:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829408.501235:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011777664:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829408.501253:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829408.501261:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829408/real 1787829408] req@ffff907d90847480 x1874670011777664/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829408.501269:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011777664/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829408.501275:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829408.501277:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829408.501280:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011777664:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829408.501304:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011777792:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829408.501315:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829408.501321:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829408/real 1787829408] req@ffff907d90845a40 x1874670011777792/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829408.501329:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011777792/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829408.501339:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829408.501341:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829408.501343:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011777792:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829408.628740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829408.628817:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829409.652666:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829409.652789:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829410.676735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829410.676820:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829411.700719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829411.700788:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829412.724700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829412.724739:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829413.620877:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829413.620885:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829413.620889:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829413.620899:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4817 00000100:00080000:2.0:1787829413.620909:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829413.620917:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829413.620948:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011777920/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829413.620976:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829413.620981:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829413.620983:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829413.620988:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4817 00000100:00080000:2.0:1787829413.620992:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829413.620996:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829413.621007:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829413.621016:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011777920:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829413.621023:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011778048/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829413.621045:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829413.621048:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829413.621050:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829413.621054:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4817 00000100:00080000:2.0:1787829413.621058:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829413.621061:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829413.621066:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829413.621077:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011778176/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829413.621086:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829413.621097:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829413.621099:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829413.621103:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4817 00000100:00080000:2.0:1787829413.621107:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829413.621112:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829413.621117:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829413.621131:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011778304/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829413.621162:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829413.621196:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829413/real 1787829413] req@ffff907d9114b840 x1874670011777920/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829413.621214:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011777920/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829413.621224:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829413.621229:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829413.621233:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011777920:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829413.621254:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011778048:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829413.621277:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829413.621289:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829413/real 1787829413] req@ffff907d9114a580 x1874670011778048/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829413.621301:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011778048/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829413.621309:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829413.621312:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829413.621316:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011778048:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829413.621322:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011778176:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829413.621344:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829413.621355:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829413/real 1787829413] req@ffff907d9114ad00 x1874670011778176/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829413.621368:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011778176/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829413.621377:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829413.621381:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829413.621385:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011778176:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829413.621391:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011778304:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829413.621409:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829413.621418:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829413/real 1787829413] req@ffff907d91149a40 x1874670011778304/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829413.621430:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011778304/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829413.621438:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829413.621441:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829413.621445:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011778304:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829413.748702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829413.748734:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829414.772719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829414.772762:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829415.796709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829415.796771:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829416.820723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829416.820771:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829417.844688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829417.844728:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829418.740697:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829418.740705:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829418.740708:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829418.740717:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4822 00000100:00080000:1.0:1787829418.740725:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829418.740731:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829418.740764:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011778432/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829418.740791:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829418.740795:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829418.740796:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829418.740799:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4822 00000100:00080000:1.0:1787829418.740802:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829418.740807:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829418.740815:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829418.740826:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011778560/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829418.740831:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011778432:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829418.740841:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829418.740844:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829418.740846:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829418.740849:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4822 00000100:00080000:1.0:1787829418.740852:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829418.740854:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829418.740857:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829418.740866:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011778688/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829418.740874:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829418.740876:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829418.740878:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829418.740880:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4822 00000100:00080000:1.0:1787829418.740883:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829418.740886:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829418.740889:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787829418.740895:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829418.740899:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011778816/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829418.740934:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829418/real 1787829418] req@ffff907d908a3840 x1874670011778432/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829418.740947:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011778432/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829418.740954:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829418.740957:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829418.740960:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011778432:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829418.740980:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011778560:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829418.740995:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829418.741003:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829418/real 1787829418] req@ffff907d908a21c0 x1874670011778560/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829418.741016:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011778560/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829418.741024:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829418.741028:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829418.741032:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011778560:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829418.741039:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011778688:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829418.741069:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829418.741078:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829418/real 1787829418] req@ffff907d908a2940 x1874670011778688/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829418.741086:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011778688/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829418.741092:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829418.741094:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829418.741096:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011778688:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829418.741100:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011778816:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829418.741112:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829418.741120:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829418/real 1787829418] req@ffff907d908a0000 x1874670011778816/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829418.741133:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011778816/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829418.741141:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829418.741144:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829418.741148:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011778816:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829418.868694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829418.868759:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829419.892699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829419.892718:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829420.916701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829420.916844:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829421.940691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829421.940753:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829422.964697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829422.964746:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829423.860681:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829423.860689:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829423.860692:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829423.860700:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4827 00000100:00080000:1.0:1787829423.860710:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829423.860715:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829423.860744:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011778944/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829423.860764:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829423.860768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829423.860770:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829423.860774:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4827 00000100:00080000:1.0:1787829423.860778:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829423.860781:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829423.860791:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829423.860804:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011779072/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829423.860805:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011778944:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787829423.860868:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787829423.860871:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829423.860874:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829423.860876:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829423.860879:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4827 00000100:00080000:1.0:1787829423.860882:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829423.860886:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829423.860891:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829423.860902:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011779200/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829423.860906:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829423/real 1787829423] req@ffff907d908a1e00 x1874670011778944/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829423.860912:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829423.860916:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829423.860918:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829423.860921:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4827 00000100:00100000:3.0:1787829423.860922:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011778944/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829423.860924:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829423.860928:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829423.860932:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1787829423.860933:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829423.860937:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829423.860941:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011779328/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829423.860942:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011778944:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829423.860964:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011779072:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829423.860997:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829423.861009:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829423/real 1787829423] req@ffff907d908a2d00 x1874670011779072/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829423.861026:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011779072/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829423.861032:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829423.861034:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829423.861036:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011779072:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829423.861040:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011779200:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829423.861061:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829423.861073:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829423/real 1787829423] req@ffff907d908a0000 x1874670011779200/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829423.861084:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011779200/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829423.861093:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829423.861096:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829423.861100:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011779200:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829423.861105:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011779328:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829423.861131:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829423.861142:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829423/real 1787829423] req@ffff907d908a2940 x1874670011779328/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829423.861152:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011779328/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829423.861166:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829423.861168:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829423.861172:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011779328:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829423.988716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829423.988908:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829425.012719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829425.012794:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829426.036719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829426.036749:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829427.060710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829427.060773:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829428.084780:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829428.084806:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829428.980752:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829428.980763:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829428.980768:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829428.980779:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4832 00000100:00080000:1.0:1787829428.980792:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829428.980799:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829428.980842:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011779456/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829428.980868:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829428.980871:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829428.980873:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829428.980876:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4832 00000100:00080000:1.0:1787829428.980880:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829428.980884:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829428.980896:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829428.980913:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011779584/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829428.980925:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829428.980929:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829428.980931:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829428.980938:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4832 00000100:00080000:1.0:1787829428.980941:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829428.980951:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829428.980953:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011779456:192.168.203.104@tcp:250:kworker.0 00000008:00100000:1.0:1787829428.980957:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829428.980975:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011779712/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829428.980986:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829428.980991:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829428.980993:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829428.980998:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4832 00000100:00080000:1.0:1787829428.981002:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829428.981008:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829428.981013:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829428.981028:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011779840/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829428.981054:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829428.981102:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829428/real 1787829428] req@ffff907d908a21c0 x1874670011779456/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829428.981122:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011779456/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829428.981131:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829428.981134:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829428.981138:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011779456:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829428.981144:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011779584:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829428.981162:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829428.981171:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829428/real 1787829428] req@ffff907d908a3840 x1874670011779584/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829428.981184:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011779584/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829428.981190:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829428.981193:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829428.981195:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011779584:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829428.981214:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011779712:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829428.981228:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829428.981240:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829428/real 1787829428] req@ffff907d908a1680 x1874670011779712/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829428.981253:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011779712/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829428.981262:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829428.981266:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829428.981269:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011779712:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829428.981276:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011779840:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829428.981298:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829428.981311:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829428/real 1787829428] req@ffff907d908a2580 x1874670011779840/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829428.981324:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011779840/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829428.981341:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829428.981345:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829428.981348:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011779840:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829429.108663:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829429.108809:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829430.132721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829430.132803:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829431.156728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829431.156803:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829432.180693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829432.180749:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829433.204714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829433.204746:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829434.100697:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829434.100705:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829434.100709:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829434.100717:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4837 00000100:00080000:1.0:1787829434.100726:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829434.100732:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829434.100764:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011779968/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829434.100787:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829434.100790:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829434.100791:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829434.100794:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4837 00000100:00080000:1.0:1787829434.100796:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829434.100799:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829434.100806:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829434.100816:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011780096/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829434.100824:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829434.100826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829434.100828:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787829434.100833:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011779968:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829434.100834:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4837 00000100:00080000:1.0:1787829434.100840:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829434.100843:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829434.100846:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829434.100855:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011780224/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829434.100862:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829434.100865:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829434.100866:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829434.100868:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4837 00000100:00080000:1.0:1787829434.100871:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829434.100873:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829434.100875:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829434.100882:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011780352/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829434.100891:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829434.100925:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829434/real 1787829434] req@ffff907d908a0f00 x1874670011779968/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829434.100938:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011779968/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829434.100946:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829434.100949:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829434.100952:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011779968:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829434.100971:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011780096:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829434.100985:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829434.100992:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829434/real 1787829434] req@ffff907d908a12c0 x1874670011780096/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829434.101000:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011780096/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829434.101006:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829434.101008:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829434.101010:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011780096:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829434.101014:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011780224:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829434.101024:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829434.101031:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829434/real 1787829434] req@ffff907d908a2580 x1874670011780224/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829434.101038:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011780224/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829434.101043:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829434.101045:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829434.101046:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011780224:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829434.101050:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011780352:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829434.101059:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829434.101065:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829434/real 1787829434] req@ffff907d908a1680 x1874670011780352/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829434.101072:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011780352/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829434.101077:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829434.101079:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829434.101081:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011780352:192.168.203.104@tcp:8:kworker.0 00010000:00010000:3.0:1787829434.228800:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829434.228822:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787829435.252688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829435.252744:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829436.276690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829436.276747:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829437.300706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829437.300746:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829438.324722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829438.324773:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829439.220734:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829439.220749:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829439.220755:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829439.220770:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4842 00000100:00080000:3.0:1787829439.220784:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829439.220794:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829439.220844:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011780480/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829439.220878:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829439.220883:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829439.220885:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829439.220890:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4842 00000100:00080000:3.0:1787829439.220895:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829439.220899:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829439.220912:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829439.220926:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011780608/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829439.220938:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829439.220943:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829439.220945:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829439.220950:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4842 00000100:00100000:2.0:1787829439.220956:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011780480:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829439.220957:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829439.220966:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829439.220972:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829439.220986:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011780736/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829439.220994:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829439.220997:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829439.220999:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829439.221001:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4842 00000100:00080000:3.0:1787829439.221004:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829439.221007:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829439.221011:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829439.221020:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011780864/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829439.221045:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829439.221103:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829439/real 1787829439] req@ffff907da562cf00 x1874670011780480/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829439.221122:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011780480/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829439.221131:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829439.221134:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829439.221138:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011780480:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829439.221144:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011780608:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829439.221167:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829439.221182:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829439/real 1787829439] req@ffff907da562e580 x1874670011780608/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829439.221196:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011780608/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829439.221205:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829439.221208:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829439.221212:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011780608:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829439.221239:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011780736:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829439.221256:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829439.221265:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829439/real 1787829439] req@ffff907da562d2c0 x1874670011780736/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829439.221273:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011780736/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829439.221283:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829439.221285:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829439.221287:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011780736:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829439.221291:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011780864:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829439.221301:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829439.221307:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829439/real 1787829439] req@ffff907da562fc00 x1874670011780864/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829439.221314:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011780864/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829439.221319:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829439.221321:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829439.221323:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011780864:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829439.348693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829439.348775:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829440.372734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829440.372758:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829441.396667:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829441.396795:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829442.420697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829442.420720:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829443.444701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829443.444749:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829444.340680:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829444.340688:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829444.340691:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829444.340700:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4848 00000100:00080000:3.0:1787829444.340709:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829444.340714:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829444.340746:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011780992/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829444.340768:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829444.340771:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829444.340773:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829444.340776:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4848 00000100:00080000:3.0:1787829444.340778:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829444.340780:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829444.340788:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829444.340796:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011781120/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829444.340804:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829444.340806:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829444.340808:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829444.340810:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4848 00000100:00080000:3.0:1787829444.340812:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829444.340815:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829444.340819:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829444.340825:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011780992:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829444.340833:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011781248/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829444.340845:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829444.340849:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829444.340851:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829444.340856:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4848 00000100:00080000:3.0:1787829444.340861:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829444.340865:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829444.340870:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829444.340883:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011781376/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829444.340945:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011781120:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829444.340997:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011781248:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829444.341026:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011781376:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829444.468688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829444.468723:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829445.492718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829445.492738:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829446.516699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829446.516739:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829447.412878:0:2337:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787829447.412928:0:2337:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829447.412945:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829447.412984:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829447.412994:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829447.413006:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:2.0:1787829447.413032:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829444/real 1787829447] req@ffff907da562c3c0 x1874670011781248/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829447.413058:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011781248/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829447.413071:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829447.413074:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829447.413078:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011781248:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829447.413083:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829444/real 1787829447] req@ffff907da562f0c0 x1874670011781376/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829447.413094:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011781376/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829447.413102:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829447.413106:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829447.413109:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011781376:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829447.413116:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829444/real 1787829447] req@ffff907da562c000 x1874670011780992/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829447.413129:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011780992/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829447.413137:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829447.413140:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829447.413144:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011780992:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829447.413151:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829444/real 1787829447] req@ffff907da562e940 x1874670011781120/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829447.413164:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011781120/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829447.413173:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829447.413177:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829447.413181:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011781120:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829447.540675:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829447.540753:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829448.564688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829448.564741:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829449.460781:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829449.460803:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829449.460808:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829449.460822:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4853 00000100:00080000:3.0:1787829449.460837:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829449.460847:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829449.460898:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011781504/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829449.460938:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829449.460947:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829449.460949:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829449.460955:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4853 00000100:00080000:3.0:1787829449.460960:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829449.460965:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829449.460978:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829449.460995:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011781632/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829449.461014:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829449.461018:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829449.461021:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829449.461026:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4853 00000100:00080000:3.0:1787829449.461030:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829449.461035:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829449.461042:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829449.461046:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011781504:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829449.461059:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011781760/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829449.461071:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829449.461076:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829449.461078:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829449.461082:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4853 00000100:00080000:3.0:1787829449.461087:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829449.461091:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829449.461096:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829449.461110:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011781888/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829449.461154:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829449.461228:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829449/real 1787829449] req@ffff907da562f840 x1874670011781504/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829449.461258:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011781504/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829449.461274:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829449.461282:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829449.461289:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011781504:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829449.461300:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011781632:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829449.461336:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829449.461353:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829449/real 1787829449] req@ffff907da562e1c0 x1874670011781632/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829449.461368:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011781632/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829449.461378:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829449.461382:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829449.461386:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011781632:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829449.461443:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011781760:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829449.461472:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829449.461499:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829449/real 1787829449] req@ffff907da562de00 x1874670011781760/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829449.461513:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011781760/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829449.461525:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829449.461530:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829449.461534:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011781760:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829449.461542:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011781888:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829449.461567:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829449.461580:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829449/real 1787829449] req@ffff907da562cb40 x1874670011781888/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829449.461593:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011781888/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829449.461602:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829449.461644:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829449.461648:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011781888:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829449.588728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829449.588806:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829450.612678:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829450.612724:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829451.636702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829451.636737:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829452.660694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829452.660710:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829453.684756:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829453.684830:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829454.580669:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829454.580677:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829454.580682:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829454.580690:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4858 00000100:00080000:3.0:1787829454.580706:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829454.580713:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829454.580746:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011782016/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829454.580772:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829454.580775:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829454.580776:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829454.580779:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4858 00000100:00080000:3.0:1787829454.580782:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829454.580784:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829454.580792:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829454.580807:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011782144/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829454.580817:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011782016:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829454.580822:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829454.580828:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829454.580830:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829454.580835:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4858 00000100:00080000:3.0:1787829454.580840:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829454.580844:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829454.580851:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829454.580863:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011782272/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829454.580871:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787829454.580881:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829454.580884:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829454.580886:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829454.580890:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4858 00000100:00080000:3.0:1787829454.580893:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829454.580895:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829454.580899:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829454.580909:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829454/real 1787829454] req@ffff907da562da40 x1874670011782016/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829454.580910:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011782400/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829454.580944:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011782016/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829454.580958:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829454.580962:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829454.580967:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011782016:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829454.580998:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011782144:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829454.581016:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829454.581026:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829454/real 1787829454] req@ffff907da562cb40 x1874670011782144/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829454.581034:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011782144/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829454.581041:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829454.581043:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829454.581045:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011782144:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829454.581049:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011782272:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829454.581061:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829454.581067:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829454/real 1787829454] req@ffff907da562de00 x1874670011782272/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829454.581074:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011782272/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829454.581087:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829454.581089:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829454.581091:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011782272:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829454.581097:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011782400:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829454.581115:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829454.581127:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829454/real 1787829454] req@ffff907da562e1c0 x1874670011782400/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829454.581139:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011782400/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829454.581156:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829454.581160:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829454.581163:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011782400:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829454.708706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829454.708746:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829455.732686:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829455.732731:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829456.756697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829456.756735:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829457.780705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829457.780767:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829458.804668:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829458.804799:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829459.700667:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829459.700675:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829459.700678:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829459.700686:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4863 00000100:00080000:3.0:1787829459.700695:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829459.700701:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829459.700731:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011782528/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829459.700753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829459.700756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829459.700757:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829459.700760:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4863 00000100:00080000:3.0:1787829459.700763:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829459.700766:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829459.700776:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829459.700789:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011782528:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829459.700792:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011782656/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829459.700807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829459.700812:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829459.700814:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829459.700819:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4863 00000100:00080000:3.0:1787829459.700824:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829459.700829:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829459.700833:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787829459.700847:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829459.700848:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011782784/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829459.700859:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829459.700862:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829459.700863:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829459.700867:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4863 00000100:00080000:3.0:1787829459.700870:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829459.700872:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829459.700876:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829459.700891:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829459/real 1787829459] req@ffff907da562f840 x1874670011782528/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829459.700891:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011782912/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829459.700931:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011782528/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829459.700939:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829459.700942:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829459.700945:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011782528:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829459.700962:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011782656:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829459.700977:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829459.700986:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829459/real 1787829459] req@ffff907da562e940 x1874670011782656/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829459.700994:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011782656/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829459.700999:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829459.701001:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829459.701004:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011782656:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829459.701008:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011782784:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829459.701019:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829459.701025:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829459/real 1787829459] req@ffff907da562c000 x1874670011782784/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829459.701032:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011782784/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829459.701038:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829459.701039:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829459.701041:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011782784:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829459.701044:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011782912:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829459.701053:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829459.701059:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829459/real 1787829459] req@ffff907da562f0c0 x1874670011782912/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829459.701066:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011782912/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829459.701071:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829459.701073:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829459.701075:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011782912:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829459.828712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829459.828805:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829460.852663:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829460.852774:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829461.876708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829461.876736:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829462.900677:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829462.900740:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829463.924705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829463.924760:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829464.820738:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829464.820749:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829464.820755:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829464.820766:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4868 00000100:00080000:0.0:1787829464.820779:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829464.820786:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829464.820830:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011783040/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829464.820856:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829464.820858:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829464.820860:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829464.820863:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4868 00000100:00080000:0.0:1787829464.820865:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829464.820868:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829464.820878:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829464.820886:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011783168/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829464.820893:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829464.820895:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829464.820896:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829464.820899:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4868 00000100:00080000:0.0:1787829464.820901:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829464.820904:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829464.820907:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829464.820913:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011783296/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829464.820926:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829464.820929:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829464.820930:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829464.820935:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4868 00000100:00080000:0.0:1787829464.820937:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829464.820937:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011783040:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829464.820944:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829464.820950:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829464.820957:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011783424/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829464.821015:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829464.821060:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829464/real 1787829464] req@ffff907d90846d00 x1874670011783040/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829464.821077:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011783040/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829464.821090:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829464.821095:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829464.821100:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011783040:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829464.821108:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011783168:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829464.821133:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829464.821148:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829464/real 1787829464] req@ffff907d90845680 x1874670011783168/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829464.821162:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011783168/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829464.821172:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829464.821175:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829464.821179:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011783168:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829464.821210:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011783296:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829464.821235:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829464.821249:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829464/real 1787829464] req@ffff907d90844780 x1874670011783296/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829464.821263:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011783296/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829464.821272:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829464.821276:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829464.821280:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011783296:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829464.821303:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011783424:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829464.821327:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829464.821341:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829464/real 1787829464] req@ffff907d908443c0 x1874670011783424/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829464.821354:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011783424/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829464.821363:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829464.821367:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829464.821370:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011783424:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829464.948790:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829464.948856:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829465.972700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829465.972850:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829466.996723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829466.996772:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829468.020683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829468.020747:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829469.044718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829469.044770:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829469.940732:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829469.940742:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829469.940747:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829469.940758:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4873 00000100:00080000:3.0:1787829469.940768:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829469.940776:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829469.940813:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011783552/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829469.940839:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829469.940843:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829469.940845:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829469.940849:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4873 00000100:00080000:3.0:1787829469.940852:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829469.940857:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829469.940867:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829469.940876:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011783552:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829469.940884:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011783680/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829469.940897:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829469.940901:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829469.940903:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829469.940908:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4873 00000100:00080000:3.0:1787829469.940925:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829469.940929:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:2.0:1787829469.940934:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:3.0:1787829469.940939:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829469.940954:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011783808/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829469.940965:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829469.940970:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1787829469.940971:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829469/real 1787829469] req@ffff907da562c3c0 x1874670011783552/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829469.940973:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829469.940983:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4873 00000100:00080000:3.0:1787829469.940990:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829469.940995:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011783552/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829469.940997:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829469.941008:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1787829469.941011:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829469.941016:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829469.941019:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011783552:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829469.941026:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011783936/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829469.941040:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011783680:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829469.941067:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829469.941081:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829469/real 1787829469] req@ffff907da562fc00 x1874670011783680/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829469.941095:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011783680/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829469.941104:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829469.941107:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829469.941111:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011783680:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829469.941118:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011783808:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829469.941142:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829469.941155:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829469/real 1787829469] req@ffff907da562d2c0 x1874670011783808/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829469.941170:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011783808/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829469.941176:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829469.941178:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829469.941180:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011783808:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829469.941192:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011783936:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829469.941218:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829469.941226:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829469/real 1787829469] req@ffff907da562e580 x1874670011783936/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829469.941234:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011783936/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829469.941239:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829469.941242:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829469.941244:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011783936:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829470.068720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829470.068752:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829471.092735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829471.092804:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829472.116694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829472.116729:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829473.140713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829473.140779:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829474.164709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829474.164756:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829475.060694:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829475.060705:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829475.060710:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829475.060720:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4878 00000100:00080000:3.0:1787829475.060731:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829475.060737:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829475.060774:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011784064/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829475.060801:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829475.060804:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829475.060806:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829475.060809:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4878 00000100:00080000:3.0:1787829475.060812:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829475.060815:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829475.060824:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829475.060836:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011784192/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829475.060841:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011784064:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829475.060848:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829475.060852:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829475.060853:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829475.060856:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4878 00000100:00080000:3.0:1787829475.060860:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829475.060862:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829475.060865:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829475.060875:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011784320/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829475.060882:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829475.060884:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829475.060886:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829475.060889:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4878 00000100:00080000:3.0:1787829475.060892:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829475.060894:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829475.060897:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787829475.060901:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829475.060907:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011784448/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829475.060939:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829475/real 1787829475] req@ffff907da562cf00 x1874670011784064/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829475.060954:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011784064/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829475.060964:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829475.060968:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829475.060972:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011784064:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829475.060995:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011784192:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829475.061020:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829475.061034:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829475/real 1787829475] req@ffff907da562c780 x1874670011784192/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829475.061048:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011784192/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829475.061058:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829475.061061:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829475.061065:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011784192:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829475.061072:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011784320:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829475.061093:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829475.061106:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829475/real 1787829475] req@ffff907da562ed00 x1874670011784320/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829475.061116:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011784320/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829475.061126:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829475.061128:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829475.061131:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011784320:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829475.061135:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011784448:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829475.061149:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829475.061156:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829475/real 1787829475] req@ffff907da562f480 x1874670011784448/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829475.061163:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011784448/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829475.061168:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829475.061170:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829475.061172:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011784448:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829475.188707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829475.188765:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829476.212699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829476.212767:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829477.236742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829477.236813:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829478.260700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829478.260795:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829479.284745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829479.284835:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829480.180689:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829480.180699:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829480.180703:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829480.180712:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4883 00000100:00080000:3.0:1787829480.180722:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829480.180729:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829480.180761:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011784576/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829480.180787:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829480.180791:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829480.180793:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829480.180796:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4883 00000100:00080000:3.0:1787829480.180800:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829480.180803:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829480.180813:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829480.180826:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011784704/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829480.180837:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011784576:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829480.180838:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829480.180845:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829480.180847:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829480.180851:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4883 00000100:00080000:3.0:1787829480.180854:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829480.180857:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829480.180861:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829480.180873:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011784832/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829480.180881:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829480.180885:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829480.180887:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829480.180891:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4883 00000800:00000100:2.0:1787829480.180893:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787829480.180905:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829480.180912:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829480.180917:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829480.180926:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011784960/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829480.180950:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829480/real 1787829480] req@ffff907da562d680 x1874670011784576/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829480.180968:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011784576/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829480.180988:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829480.180992:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829480.180996:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011784576:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829480.181016:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011784704:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829480.181036:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829480.181045:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829480/real 1787829480] req@ffff907da562f480 x1874670011784704/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829480.181053:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011784704/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829480.181058:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829480.181060:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829480.181063:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011784704:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829480.181068:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011784832:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829480.181081:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829480.181090:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829480/real 1787829480] req@ffff907da562ed00 x1874670011784832/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829480.181100:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011784832/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829480.181108:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829480.181111:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829480.181114:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011784832:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829480.181120:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011784960:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829480.181151:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829480.181162:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829480/real 1787829480] req@ffff907da562c780 x1874670011784960/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829480.181173:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011784960/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829480.181196:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829480.181199:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829480.181201:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011784960:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829480.308679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829480.308743:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829481.332693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829481.332749:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829482.356705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829482.356755:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829483.380710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829483.380803:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829484.404788:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829484.404814:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:3.0:1787829485.300700:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829485.300707:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829485.300710:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829485.300717:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4889 00000100:00080000:3.0:1787829485.300725:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829485.300730:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829485.300755:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011785088/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829485.300774:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829485.300776:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829485.300777:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829485.300780:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4889 00000100:00080000:3.0:1787829485.300782:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829485.300784:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829485.300791:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829485.300799:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011785216/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829485.300805:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829485.300807:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829485.300809:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829485.300811:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4889 00000100:00080000:3.0:1787829485.300812:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829485.300815:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829485.300818:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829485.300821:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011785088:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829485.300827:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011785344/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829485.300834:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829485.300836:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829485.300837:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829485.300839:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4889 00000100:00080000:3.0:1787829485.300841:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829485.300843:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829485.300846:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829485.300853:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011785472/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829485.300882:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829485.300923:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829485/real 1787829485] req@ffff907da562cf00 x1874670011785088/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829485.300942:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011785088/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829485.300952:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829485.300956:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829485.300961:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011785088:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829485.300969:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011785216:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829485.300999:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829485.301015:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829485/real 1787829485] req@ffff907da562e580 x1874670011785216/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829485.301029:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011785216/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829485.301038:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829485.301042:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829485.301047:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011785216:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829485.301073:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011785344:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829485.301095:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829485.301108:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829485/real 1787829485] req@ffff907da562d2c0 x1874670011785344/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829485.301118:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011785344/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829485.301125:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829485.301128:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829485.301131:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011785344:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829485.301137:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011785472:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829485.301178:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829485.301192:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829485/real 1787829485] req@ffff907da562fc00 x1874670011785472/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829485.301207:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011785472/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829485.301216:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829485.301219:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829485.301223:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011785472:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829485.428692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829485.428741:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829486.452716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829486.452760:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829487.476709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829487.476714:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829488.500702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829488.500751:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829489.524681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829489.524727:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829490.420679:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829490.420687:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829490.420691:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829490.420699:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4894 00000100:00080000:3.0:1787829490.420709:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829490.420714:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829490.420742:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011785600/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829490.420759:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829490.420761:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829490.420763:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829490.420765:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4894 00000100:00080000:3.0:1787829490.420767:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829490.420769:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829490.420776:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829490.420789:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011785728/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829490.420793:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011785600:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829490.420806:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829490.420810:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829490.420812:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829490.420817:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4894 00000100:00080000:3.0:1787829490.420821:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829490.420825:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829490.420831:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829490.420839:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011785856/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829490.420846:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787829490.420848:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829490.420851:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829490.420852:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829490.420856:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4894 00000100:00080000:3.0:1787829490.420860:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829490.420863:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829490.420866:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829490.420874:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011785984/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829490.420882:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829490/real 1787829490] req@ffff907da562c3c0 x1874670011785600/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829490.420896:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011785600/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829490.420902:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829490.420904:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829490.420907:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011785600:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829490.420922:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011785728:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829490.420936:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829490.420943:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829490/real 1787829490] req@ffff907da562f0c0 x1874670011785728/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829490.420950:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011785728/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829490.420955:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829490.420957:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829490.420960:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011785728:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829490.420964:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011785856:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829490.420982:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829490.420992:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829490/real 1787829490] req@ffff907da562c000 x1874670011785856/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829490.421004:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011785856/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829490.421012:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829490.421014:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829490.421017:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011785856:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829490.421023:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011785984:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829490.421040:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829490.421051:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829490/real 1787829490] req@ffff907da562e940 x1874670011785984/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829490.421061:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011785984/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829490.421074:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829490.421077:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829490.421080:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011785984:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829490.548670:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829490.548711:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829491.572684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829491.572727:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829492.596680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829492.596722:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829493.620669:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829493.620714:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829494.644713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829494.644880:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829495.540755:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829495.540766:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829495.540770:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829495.540780:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4899 00000100:00080000:1.0:1787829495.540791:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829495.540798:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829495.540838:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011786112/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829495.540867:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829495.540870:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829495.540872:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829495.540875:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4899 00000100:00080000:1.0:1787829495.540878:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829495.540881:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829495.540891:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829495.540901:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011786240/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829495.540910:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829495.540912:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829495.540914:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829495.540920:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4899 00000100:00100000:2.0:1787829495.540920:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011786112:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829495.540924:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829495.540929:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829495.540933:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829495.540942:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011786368/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829495.540948:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829495.540951:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829495.540952:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829495.540955:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4899 00000100:00080000:1.0:1787829495.540957:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829495.540960:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829495.540962:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829495.540970:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011786496/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829495.540992:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829495.541034:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829495/real 1787829495] req@ffff907d908a3840 x1874670011786112/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829495.541049:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011786112/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829495.541056:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829495.541060:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829495.541063:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011786112:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829495.541100:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011786240:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829495.541119:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829495.541128:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829495/real 1787829495] req@ffff907d908a21c0 x1874670011786240/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829495.541137:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011786240/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829495.541143:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829495.541145:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829495.541148:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011786240:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829495.541152:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011786368:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829495.541164:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829495.541171:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829495/real 1787829495] req@ffff907d908a2940 x1874670011786368/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829495.541178:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011786368/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829495.541183:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829495.541185:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829495.541187:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011786368:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829495.541191:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011786496:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829495.541201:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829495.541207:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829495/real 1787829495] req@ffff907d908a0000 x1874670011786496/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829495.541222:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011786496/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829495.541227:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829495.541230:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829495.541232:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011786496:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829495.668724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829495.668810:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829496.692735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829496.692798:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829497.716720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829497.716788:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829498.740736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829498.740799:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829499.764701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829499.764732:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829500.660736:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829500.660750:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829500.660756:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829500.660771:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4904 00000100:00080000:3.0:1787829500.660783:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829500.660791:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829500.660832:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011786624/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829500.660863:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829500.660868:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829500.660870:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829500.660877:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4904 00000100:00080000:3.0:1787829500.660882:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829500.660888:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829500.660895:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011786624:192.168.203.104@tcp:250:kworker.0 00000008:00100000:3.0:1787829500.660899:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829500.660914:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011786752/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829500.660923:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829500.660926:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829500.660929:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829500.660933:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4904 00000100:00080000:3.0:1787829500.660958:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829500.660964:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:2.0:1787829500.660970:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:3.0:1787829500.660971:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829500.660988:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011786880/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829500.661000:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829500.661004:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829500.661006:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829500.661010:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4904 00000100:00080000:3.0:1787829500.661014:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829500.661020:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829500.661021:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829500/real 1787829500] req@ffff907da562f840 x1874670011786624/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787829500.661025:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829500.661051:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011787008/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829500.661058:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011786624/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829500.661075:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829500.661081:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829500.661086:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011786624:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829500.661118:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011786752:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829500.661146:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829500.661162:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829500/real 1787829500] req@ffff907da562e1c0 x1874670011786752/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829500.661176:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011786752/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829500.661186:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829500.661190:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829500.661194:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011786752:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829500.661202:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011786880:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829500.661226:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829500.661239:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829500/real 1787829500] req@ffff907da562de00 x1874670011786880/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829500.661252:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011786880/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829500.661262:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829500.661266:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829500.661269:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011786880:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829500.661276:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011787008:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829500.661298:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829500.661310:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829500/real 1787829500] req@ffff907da562cb40 x1874670011787008/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829500.661319:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011787008/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829500.661325:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829500.661327:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829500.661337:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011787008:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829500.788726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829500.788788:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829501.812668:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829501.812756:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829502.836747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829502.836818:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829503.860740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829503.860815:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829504.884734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829504.884734:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829505.780736:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829505.780745:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829505.780750:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829505.780761:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4909 00000100:00080000:0.0:1787829505.780772:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829505.780779:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829505.780828:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011787136/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829505.780853:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829505.780856:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829505.780858:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829505.780861:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4909 00000100:00080000:0.0:1787829505.780863:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829505.780865:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829505.780875:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829505.780886:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011787264/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829505.780893:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829505.780895:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829505.780897:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829505.780899:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4909 00000100:00080000:0.0:1787829505.780901:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829505.780904:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829505.780908:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829505.780910:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011787136:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829505.780917:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011787392/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829505.780925:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829505.780927:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829505.780929:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829505.780931:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4909 00000100:00080000:0.0:1787829505.780933:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829505.780935:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829505.780938:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829505.780946:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011787520/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829505.780980:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829505.781038:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829505/real 1787829505] req@ffff907d90845e00 x1874670011787136/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829505.781053:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011787136/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829505.781062:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829505.781065:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829505.781068:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011787136:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829505.781074:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011787264:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829505.781091:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829505.781099:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829505/real 1787829505] req@ffff907d90844000 x1874670011787264/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829505.781107:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011787264/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829505.781113:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829505.781114:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829505.781116:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011787264:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829505.781137:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011787392:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829505.781151:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829505.781158:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829505/real 1787829505] req@ffff907d908452c0 x1874670011787392/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829505.781166:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011787392/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829505.781172:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829505.781174:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829505.781176:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011787392:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829505.781179:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011787520:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829505.781188:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829505.781194:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829505/real 1787829505] req@ffff907d90844b40 x1874670011787520/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829505.781201:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011787520/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829505.781206:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829505.781208:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829505.781210:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011787520:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829505.908722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829505.908772:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829506.932740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829506.932806:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829507.956696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829507.956742:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829508.980681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829508.980710:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829510.004719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829510.004809:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829510.900696:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829510.900707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829510.900711:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829510.900722:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4914 00000100:00080000:1.0:1787829510.900733:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829510.900740:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829510.900786:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011787648/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829510.900815:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829510.900819:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829510.900821:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829510.900825:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4914 00000100:00080000:1.0:1787829510.900830:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829510.900835:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829510.900847:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829510.900860:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011787776/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829510.900868:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011787648:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829510.900871:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829510.900878:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829510.900880:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829510.900884:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4914 00000100:00080000:1.0:1787829510.900886:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829510.900889:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829510.900893:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829510.900903:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011787904/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829510.900911:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829510.900913:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829510.900915:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829510.900918:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4914 00000100:00080000:1.0:1787829510.900920:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829510.900922:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829510.900925:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829510.900932:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011788032/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829510.900990:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011787776:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829510.901022:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011787904:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829510.901040:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011788032:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829511.028699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829511.028736:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829512.052748:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829512.052815:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829513.076717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829513.076746:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787829513.972876:0:2338:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:0.0:1787829513.972922:0:2338:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787829513.972939:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787829513.972997:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787829513.973009:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787829513.973019:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:2.0:1787829513.973083:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829510/real 1787829513] req@ffff907d908a1e00 x1874670011787776/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829513.973108:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011787776/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829513.973126:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829513.973132:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829513.973142:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011787776:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829513.973150:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829510/real 1787829513] req@ffff907d908a0000 x1874670011787904/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829513.973163:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011787904/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829513.973171:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829513.973175:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829513.973178:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011787904:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829513.973184:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829510/real 1787829513] req@ffff907d908a2940 x1874670011788032/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829513.973196:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011788032/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829513.973203:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829513.973205:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829513.973208:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011788032:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829513.973213:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829510/real 1787829513] req@ffff907d908a2d00 x1874670011787648/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829513.973223:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011787648/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829513.973230:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829513.973232:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829513.973235:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011787648:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829514.100712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829514.100824:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829515.124683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829515.124754:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829516.020762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829516.020776:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829516.020781:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829516.020791:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4919 00000100:00080000:3.0:1787829516.020804:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829516.020813:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829516.020856:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011788160/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829516.020893:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829516.020898:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829516.020900:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829516.020906:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4919 00000100:00080000:3.0:1787829516.020910:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829516.020925:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829516.020952:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829516.020959:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011788160:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829516.020977:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011788288/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829516.020999:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829516.021003:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829516.021005:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829516.021011:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4919 00000100:00080000:3.0:1787829516.021036:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829516.021042:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:2.0:1787829516.021052:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:3.0:1787829516.021054:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829516.021078:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011788416/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829516.021089:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829516.021094:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829516.021097:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829516.021104:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4919 00000100:00080000:3.0:1787829516.021110:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829516.021115:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829516/real 1787829516] req@ffff907da562da40 x1874670011788160/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829516.021122:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829516.021135:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829516.021152:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011788160/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829516.021158:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011788544/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829516.021206:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829516.021218:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829516.021225:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011788160:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829516.021255:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011788288:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829516.021281:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829516.021294:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829516/real 1787829516] req@ffff907da562cb40 x1874670011788288/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829516.021313:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011788288/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829516.021321:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829516.021324:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829516.021341:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011788288:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829516.021347:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011788416:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829516.021367:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829516.021379:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829516/real 1787829516] req@ffff907da562de00 x1874670011788416/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829516.021389:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011788416/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829516.021396:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829516.021400:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829516.021403:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011788416:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829516.021409:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011788544:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829516.021427:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829516.021437:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829516/real 1787829516] req@ffff907da562e1c0 x1874670011788544/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829516.021449:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011788544/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829516.021456:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829516.021459:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829516.021463:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011788544:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829516.148696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829516.148712:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829517.172705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829517.172798:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829518.196752:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829518.196827:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829519.220674:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829519.220746:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829520.244690:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829520.244800:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829521.140694:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829521.140704:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829521.140708:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829521.140717:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4924 00000100:00080000:1.0:1787829521.140727:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829521.140735:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829521.140768:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011788672/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829521.140796:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829521.140800:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829521.140802:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829521.140807:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4924 00000100:00080000:1.0:1787829521.140811:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829521.140816:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829521.140823:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011788672:192.168.203.104@tcp:250:kworker.0 00000008:00100000:1.0:1787829521.140825:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829521.140843:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011788800/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829521.140856:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829521.140860:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829521.140862:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829521.140865:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4924 00000100:00080000:1.0:1787829521.140868:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829521.140874:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:3.0:1787829521.140876:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:1.0:1787829521.140881:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829521.140905:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011788928/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829521.140917:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829521.140920:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1787829521.140922:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829521/real 1787829521] req@ffff907d908a21c0 x1874670011788672/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829521.140924:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829521.140933:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4924 00000100:00080000:1.0:1787829521.140939:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787829521.140941:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011788672/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829521.140946:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00080000:3.0:1787829521.140952:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1787829521.140955:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1787829521.140957:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829521.140967:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011789056/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829521.141005:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011788672:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829521.141022:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011788800:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829521.141045:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829521.141057:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829521/real 1787829521] req@ffff907d908a3840 x1874670011788800/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829521.141070:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011788800/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829521.141078:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829521.141081:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829521.141084:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011788800:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829521.141090:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011788928:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829521.141110:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829521.141121:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829521/real 1787829521] req@ffff907d908a1680 x1874670011788928/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829521.141254:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011788928/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829521.141264:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829521.141268:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829521.141273:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011788928:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829521.141321:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011789056:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829521.141346:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829521.141358:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829521/real 1787829521] req@ffff907d908a2580 x1874670011789056/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829521.141369:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011789056/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829521.141378:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829521.141381:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829521.141384:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011789056:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829521.268699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829521.268747:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829522.292734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829522.292791:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829523.316728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829523.316750:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829524.340744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829524.340803:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829525.364730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829525.364794:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829526.260759:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829526.260769:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829526.260774:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829526.260788:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4930 00000100:00080000:3.0:1787829526.260799:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829526.260806:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829526.260844:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011789184/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829526.260885:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829526.260888:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829526.260890:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829526.260893:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4930 00000100:00080000:3.0:1787829526.260896:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829526.260899:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829526.260909:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829526.260921:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011789312/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829526.260930:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829526.260932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829526.260934:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829526.260939:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011789184:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829526.260940:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4930 00000100:00080000:3.0:1787829526.260944:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829526.260950:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829526.260954:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829526.260962:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011789440/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829526.260969:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829526.260972:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829526.260974:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829526.260976:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4930 00000100:00080000:3.0:1787829526.260979:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829526.260981:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829526.260984:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829526.260992:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011789568/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829526.261015:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829526.261054:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829526/real 1787829526] req@ffff907da562f840 x1874670011789184/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829526.261068:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011789184/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829526.261077:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829526.261080:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829526.261083:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011789184:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829526.261106:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011789312:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829526.261121:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829526.261129:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829526/real 1787829526] req@ffff907da562e940 x1874670011789312/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829526.261137:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011789312/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829526.261142:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829526.261144:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829526.261147:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011789312:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829526.261150:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011789440:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829526.261162:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829526.261167:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829526/real 1787829526] req@ffff907da562c000 x1874670011789440/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829526.261175:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011789440/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829526.261180:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829526.261182:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829526.261184:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011789440:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829526.261187:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011789568:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829526.261197:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829526.261203:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829526/real 1787829526] req@ffff907da562f0c0 x1874670011789568/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829526.261210:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011789568/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829526.261215:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829526.261217:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829526.261219:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011789568:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829526.388727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829526.388786:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829527.412740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829527.412836:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829528.436722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829528.436815:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829529.460736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829529.460799:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829530.484696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829530.484728:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829531.380703:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829531.380718:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829531.380723:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829531.380735:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4935 00000100:00080000:1.0:1787829531.380750:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829531.380759:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829531.380806:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011789696/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829531.380845:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829531.380848:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829531.380850:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829531.380853:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4935 00000100:00080000:1.0:1787829531.380857:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829531.380860:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829531.380871:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829531.380876:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011789696:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829531.380886:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011789824/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829531.380900:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829531.380903:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829531.380904:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829531.380907:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4935 00000100:00080000:1.0:1787829531.380910:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829531.380913:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829531.380916:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829531.380926:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011789952/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829531.380933:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829531.380935:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829531.380937:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829531.380939:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4935 00000100:00080000:1.0:1787829531.380941:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829531.380945:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:2.0:1787829531.380946:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:1.0:1787829531.380949:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829531.380958:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011790080/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829531.380993:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829531/real 1787829531] req@ffff907d908a12c0 x1874670011789696/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829531.381008:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011789696/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829531.381016:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829531.381021:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829531.381025:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011789696:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829531.381056:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011789824:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829531.381083:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829531.381095:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829531/real 1787829531] req@ffff907d908a0f00 x1874670011789824/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829531.381104:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011789824/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829531.381110:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829531.381113:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829531.381115:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011789824:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829531.381120:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011789952:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829531.381133:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829531.381143:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829531/real 1787829531] req@ffff907d908a2580 x1874670011789952/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829531.381152:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011789952/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829531.381158:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829531.381161:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829531.381163:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011789952:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829531.381167:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011790080:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829531.381181:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829531.381192:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829531/real 1787829531] req@ffff907d908a1680 x1874670011790080/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829531.381214:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011790080/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829531.381224:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829531.381228:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829531.381231:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011790080:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829531.508761:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829531.508819:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829532.532716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829532.532775:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829533.556764:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829533.556854:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829534.580694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829534.580745:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829535.604726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829535.604746:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829536.500710:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829536.500717:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829536.500720:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829536.500727:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4940 00000100:00080000:0.0:1787829536.500735:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829536.500740:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829536.500767:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011790208/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829536.500786:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829536.500798:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829536.500800:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829536.500803:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4940 00000100:00080000:0.0:1787829536.500806:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829536.500810:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829536.500818:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829536.500829:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011790336/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829536.500837:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829536.500839:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829536.500840:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829536.500847:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4940 00000100:00100000:2.0:1787829536.500847:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011790208:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829536.500849:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829536.500855:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829536.500859:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829536.500866:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011790464/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829536.500871:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829536.500873:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829536.500874:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829536.500876:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4940 00000100:00080000:0.0:1787829536.500878:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829536.500880:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829536.500883:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829536.500890:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011790592/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829536.500910:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829536.500948:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829536/real 1787829536] req@ffff907d90847c00 x1874670011790208/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829536.500965:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011790208/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829536.500974:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829536.500978:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829536.500982:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011790208:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829536.501003:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011790336:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829536.501025:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829536.501036:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829536/real 1787829536] req@ffff907d90846580 x1874670011790336/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829536.501043:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011790336/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829536.501048:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829536.501050:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829536.501052:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011790336:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829536.501056:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011790464:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829536.501067:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829536.501073:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829536/real 1787829536] req@ffff907d908470c0 x1874670011790464/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829536.501079:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011790464/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829536.501084:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829536.501085:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829536.501087:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011790464:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829536.501090:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011790592:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829536.501098:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829536.501103:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829536/real 1787829536] req@ffff907d90844b40 x1874670011790592/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829536.501108:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011790592/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829536.501113:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829536.501115:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829536.501116:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011790592:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829536.628674:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829536.628723:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829537.652704:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829537.652746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787829538.676696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829538.676810:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829539.700728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829539.700799:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829540.724682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829540.724775:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829541.620745:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829541.620767:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829541.620771:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829541.620782:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4945 00000100:00080000:3.0:1787829541.620794:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829541.620801:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829541.620843:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011790720/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829541.620872:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829541.620875:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829541.620876:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829541.620879:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4945 00000100:00080000:3.0:1787829541.620882:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829541.620884:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829541.620894:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829541.620904:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011790848/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829541.620913:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829541.620916:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829541.620917:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829541.620923:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4945 00000100:00100000:2.0:1787829541.620925:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011790720:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829541.620926:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829541.620932:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829541.620936:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829541.620944:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011790976/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829541.620951:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829541.620954:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829541.620956:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829541.620958:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4945 00000100:00080000:3.0:1787829541.620962:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829541.620964:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829541.620968:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829541.620977:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011791104/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829541.620993:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829541.621043:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829541/real 1787829541] req@ffff907da562c3c0 x1874670011790720/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829541.621058:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011790720/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829541.621067:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829541.621070:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829541.621073:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011790720:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829541.621101:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011790848:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829541.621118:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829541.621127:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829541/real 1787829541] req@ffff907da562fc00 x1874670011790848/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829541.621136:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011790848/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829541.621143:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829541.621145:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829541.621148:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011790848:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829541.621151:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011790976:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829541.621164:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829541.621170:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829541/real 1787829541] req@ffff907da562d2c0 x1874670011790976/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829541.621178:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011790976/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829541.621184:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829541.621185:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829541.621187:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011790976:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829541.621191:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011791104:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829541.621201:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829541.621207:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829541/real 1787829541] req@ffff907da562e580 x1874670011791104/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829541.621214:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011791104/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829541.621219:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829541.621221:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829541.621224:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011791104:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829541.748725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829541.748791:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829542.772735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829542.772814:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829543.796697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829543.796766:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829544.820757:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829544.820809:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829545.844717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829545.844792:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829546.740668:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829546.740677:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829546.740681:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829546.740689:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4950 00000100:00080000:3.0:1787829546.740698:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829546.740703:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829546.740747:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011791232/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829546.740773:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829546.740776:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829546.740778:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829546.740781:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4950 00000100:00080000:3.0:1787829546.740783:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829546.740786:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829546.740794:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829546.740804:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011791360/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829546.740814:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829546.740818:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829546.740820:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829546.740825:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4950 00000100:00080000:3.0:1787829546.740829:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829546.740835:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829546.740846:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011791232:192.168.203.104@tcp:250:kworker.0 00000008:00100000:3.0:1787829546.740851:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829546.740875:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011791488/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829546.740888:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829546.740892:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829546.740894:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829546.740905:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4950 00000100:00080000:3.0:1787829546.740917:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829546.740922:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:2.0:1787829546.740929:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:3.0:1787829546.740935:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829546.740954:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011791616/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829546.740983:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829546/real 1787829546] req@ffff907da562cf00 x1874670011791232/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829546.741000:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011791232/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829546.741009:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829546.741012:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829546.741016:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011791232:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829546.741022:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011791360:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829546.741038:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829546.741046:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829546/real 1787829546] req@ffff907da562c780 x1874670011791360/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829546.741054:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011791360/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829546.741060:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829546.741062:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829546.741064:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011791360:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829546.741089:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011791488:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829546.741109:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829546.741122:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829546/real 1787829546] req@ffff907da562ed00 x1874670011791488/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829546.741135:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011791488/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829546.741143:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829546.741147:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829546.741151:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011791488:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829546.741157:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011791616:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829546.741179:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829546.741192:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829546/real 1787829546] req@ffff907da562f480 x1874670011791616/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829546.741201:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011791616/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829546.741207:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829546.741210:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829546.741212:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011791616:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829546.868732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829546.868801:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829547.892706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829547.892798:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829548.916777:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829548.916845:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829549.940745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829549.940810:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829550.964787:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829550.964845:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829551.860803:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829551.860814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829551.860820:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829551.860832:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4955 00000100:00080000:1.0:1787829551.860845:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829551.860852:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829551.860900:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011791744/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829551.860933:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829551.860936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829551.860938:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829551.860942:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4955 00000100:00080000:1.0:1787829551.860946:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829551.860951:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829551.860964:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829551.860982:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011791872/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829551.860989:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011791744:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829551.861001:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829551.861006:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829551.861008:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829551.861014:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4955 00000100:00080000:1.0:1787829551.861018:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829551.861023:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829551.861030:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829551.861044:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011792000/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829551.861057:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829551.861060:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829551.861061:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829551.861064:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4955 00000100:00080000:1.0:1787829551.861068:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787829551.861069:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787829551.861076:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829551.861084:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829551.861097:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011792128/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829551.861123:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829551/real 1787829551] req@ffff907d908a3840 x1874670011791744/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829551.861140:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011791744/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829551.861150:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829551.861154:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829551.861156:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011791744:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829551.861179:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011791872:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829551.861195:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829551.861203:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829551/real 1787829551] req@ffff907d908a21c0 x1874670011791872/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829551.861211:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011791872/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829551.861217:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829551.861219:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829551.861221:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011791872:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829551.861225:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011792000:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829551.861237:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829551.861243:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829551/real 1787829551] req@ffff907d908a2d00 x1874670011792000/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829551.861251:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011792000/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829551.861256:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829551.861258:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829551.861260:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011792000:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829551.861265:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011792128:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829551.861283:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829551.861307:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829551/real 1787829551] req@ffff907d908a2940 x1874670011792128/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829551.861320:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011792128/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829551.861326:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829551.861328:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829551.861331:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011792128:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829551.988737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829551.988806:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829553.012850:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829553.012890:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787829554.036716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829554.036771:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829555.060771:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829555.060862:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829556.084711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829556.084781:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829556.980764:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829556.980779:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829556.980784:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829556.980799:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4960 00000100:00080000:1.0:1787829556.980813:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829556.980824:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829556.980873:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011792256/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829556.980912:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829556.980915:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829556.980917:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829556.980920:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4960 00000100:00080000:1.0:1787829556.980924:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829556.980928:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829556.980939:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829556.980953:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011792384/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829556.980962:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829556.980964:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829556.980966:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829556.980972:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4960 00000100:00100000:0.0:1787829556.980976:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011792256:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829556.980977:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829556.980986:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829556.980994:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829556.981012:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011792512/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829556.981023:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829556.981028:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829556.981030:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829556.981035:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4960 00000100:00080000:1.0:1787829556.981041:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829556.981047:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829556.981054:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787829556.981063:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829556.981069:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011792640/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829556.981113:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829556/real 1787829556] req@ffff907d908a0000 x1874670011792256/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829556.981130:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011792256/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829556.981140:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829556.981144:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829556.981149:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011792256:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829556.981177:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011792384:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829556.981208:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829556.981226:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829556/real 1787829556] req@ffff907d908a1e00 x1874670011792384/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829556.981240:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011792384/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829556.981250:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829556.981254:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829556.981258:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011792384:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829556.981267:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011792512:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829556.981287:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829556.981306:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829556/real 1787829556] req@ffff907d908a2940 x1874670011792512/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829556.981316:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011792512/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829556.981323:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829556.981325:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829556.981328:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011792512:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829556.981332:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011792640:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829556.981347:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829556.981357:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829556/real 1787829556] req@ffff907d908a2d00 x1874670011792640/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829556.981368:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011792640/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829556.981373:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829556.981376:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829556.981379:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011792640:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829557.108747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829557.108767:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829558.132705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829558.132772:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829559.156744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829559.156810:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829560.180700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829560.180772:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829561.204717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829561.204781:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829562.100732:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829562.100739:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829562.100743:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829562.100752:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4965 00000100:00080000:3.0:1787829562.100762:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829562.100769:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829562.100804:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011792768/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829562.100829:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829562.100832:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829562.100833:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829562.100836:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4965 00000100:00080000:3.0:1787829562.100839:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829562.100842:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829562.100849:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829562.100861:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011792896/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829562.100870:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829562.100873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829562.100875:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829562.100880:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4965 00000100:00100000:0.0:1787829562.100883:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011792768:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829562.100883:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829562.100887:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829562.100894:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829562.100901:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011793024/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829562.100909:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829562.100911:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829562.100912:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829562.100915:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4965 00000100:00080000:3.0:1787829562.100917:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829562.100919:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829562.100922:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829562.100930:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011793152/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829562.100954:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829562.100996:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829562/real 1787829562] req@ffff907da562d680 x1874670011792768/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829562.101016:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011792768/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829562.101024:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829562.101027:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829562.101030:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011792768:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829562.101036:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011792896:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829562.101054:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829562.101065:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829562/real 1787829562] req@ffff907da562f480 x1874670011792896/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829562.101077:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011792896/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829562.101093:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829562.101096:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829562.101100:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011792896:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829562.101123:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011793024:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829562.101140:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829562.101147:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829562/real 1787829562] req@ffff907da562ed00 x1874670011793024/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829562.101155:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011793024/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829562.101160:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829562.101162:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829562.101164:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011793024:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829562.101167:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011793152:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829562.101177:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829562.101183:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829562/real 1787829562] req@ffff907da562c780 x1874670011793152/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829562.101190:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011793152/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829562.101195:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829562.101197:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829562.101199:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011793152:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829562.228730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829562.228771:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829563.252687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829563.252785:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829564.276719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829564.276795:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829565.300717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829565.300760:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829566.324739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829566.324793:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829567.220747:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829567.220762:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829567.220767:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829567.220781:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4970 00000100:00080000:3.0:1787829567.220795:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829567.220804:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829567.220867:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011793280/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829567.220902:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829567.220905:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829567.220907:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829567.220912:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4970 00000100:00080000:3.0:1787829567.220916:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829567.220920:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829567.220932:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829567.220948:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011793408/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829567.220957:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011793280:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829567.220965:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829567.220969:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829567.220971:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829567.220976:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4970 00000100:00080000:3.0:1787829567.220981:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829567.220985:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829567.220991:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829567.221015:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011793536/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829567.221027:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787829567.221032:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787829567.221034:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829567.221038:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829567.221045:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4970 00000100:00080000:3.0:1787829567.221051:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829567.221056:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829567.221062:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829567.221080:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829567/real 1787829567] req@ffff907da562cf00 x1874670011793280/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829567.221086:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011793664/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829567.221104:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011793280/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829567.221114:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829567.221119:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829567.221125:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011793280:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829567.221156:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011793408:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829567.221182:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829567.221193:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829567/real 1787829567] req@ffff907da562e580 x1874670011793408/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829567.221211:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011793408/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829567.221218:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829567.221220:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829567.221222:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011793408:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829567.221227:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011793536:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829567.221244:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829567.221251:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829567/real 1787829567] req@ffff907da562d2c0 x1874670011793536/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829567.221259:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011793536/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829567.221265:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829567.221266:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829567.221268:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011793536:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829567.221272:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011793664:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829567.221283:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829567.221289:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829567/real 1787829567] req@ffff907da562fc00 x1874670011793664/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829567.221296:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011793664/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829567.221302:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829567.221304:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829567.221305:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011793664:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829567.348723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829567.348800:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829568.372730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829568.372799:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829569.396695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829569.396762:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829570.420729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829570.420802:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829571.444727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829571.444731:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829572.340711:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829572.340723:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829572.340726:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829572.340735:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4976 00000100:00080000:2.0:1787829572.340745:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829572.340751:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829572.340783:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011793792/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829572.340805:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829572.340808:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829572.340810:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829572.340813:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4976 00000100:00080000:2.0:1787829572.340815:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829572.340817:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829572.340827:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829572.340837:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011793920/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829572.340849:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829572.340851:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829572.340852:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829572.340856:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4976 00000100:00080000:2.0:1787829572.340859:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829572.340868:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829572.340869:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011793792:192.168.203.104@tcp:250:kworker.0 00000008:00100000:2.0:1787829572.340873:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829572.340891:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011794048/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829572.340897:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829572.340900:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829572.340901:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829572.340904:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4976 00000100:00080000:2.0:1787829572.340906:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829572.340908:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829572.340911:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829572.340918:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011794176/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829572.340963:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829572.341010:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829572/real 1787829572] req@ffff907d9114b480 x1874670011793792/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829572.341031:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011793792/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829572.341040:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829572.341043:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829572.341046:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011793792:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829572.341069:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011793920:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829572.341089:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829572.341103:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829572/real 1787829572] req@ffff907d91149680 x1874670011793920/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829572.341116:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011793920/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829572.341127:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829572.341130:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829572.341134:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011793920:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829572.341141:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011794048:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829572.341160:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829572.341169:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829572/real 1787829572] req@ffff907d9114bc00 x1874670011794048/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829572.341177:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011794048/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829572.341188:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829572.341190:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829572.341193:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011794048:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829572.341197:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011794176:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829572.341208:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829572.341214:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829572/real 1787829572] req@ffff907d911483c0 x1874670011794176/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829572.341222:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011794176/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829572.341227:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829572.341229:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829572.341231:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011794176:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829572.468699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829572.468743:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829573.492685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829573.492815:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829574.516739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829574.516812:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829575.540679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829575.540817:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829576.564717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829576.564795:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829577.460721:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829577.460730:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829577.460735:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829577.460746:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4981 00000100:00080000:0.0:1787829577.460756:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829577.460763:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829577.460799:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011794304/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829577.460830:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829577.460835:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829577.460837:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829577.460841:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4981 00000100:00080000:0.0:1787829577.460845:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829577.460850:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829577.460861:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829577.460872:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011794304:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829577.460877:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011794432/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829577.460891:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829577.460895:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829577.460897:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829577.460901:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4981 00000100:00080000:0.0:1787829577.460905:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829577.460909:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829577.460914:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829577.460927:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011794560/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829577.460937:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829577.460941:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829577.460943:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829577.460948:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4981 00000100:00080000:0.0:1787829577.460953:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829577.460957:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829577.460964:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829577.460966:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011794432:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829577.460980:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011794688/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829577.461010:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011794560:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829577.461034:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011794688:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829577.588693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829577.588742:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829578.612694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829578.612718:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829579.636782:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829579.636852:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829580.532854:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787829580.532894:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829580.532912:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829580.532949:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829580.532958:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829580.532968:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:2.0:1787829580.533030:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829577/real 1787829580] req@ffff907d908443c0 x1874670011794688/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829580.533068:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011794688/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829580.533084:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829580.533091:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829580.533097:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011794688:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829580.533108:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829577/real 1787829580] req@ffff907d90844000 x1874670011794432/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829580.533122:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011794432/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829580.533132:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829580.533136:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829580.533140:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011794432:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829580.533149:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829577/real 1787829580] req@ffff907d90845e00 x1874670011794560/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829580.533162:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011794560/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829580.533172:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829580.533175:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829580.533178:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011794560:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829580.533187:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829577/real 1787829580] req@ffff907d908452c0 x1874670011794304/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829580.533208:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011794304/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829580.533217:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829580.533221:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829580.533224:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011794304:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829580.660756:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829580.660843:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829581.684740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829581.684800:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829582.580699:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829582.580711:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829582.580715:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829582.580727:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4986 00000100:00080000:0.0:1787829582.580740:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829582.580750:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829582.580799:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011794816/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829582.580832:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829582.580837:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829582.580839:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829582.580844:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4986 00000100:00080000:0.0:1787829582.580848:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829582.580852:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829582.580864:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829582.580878:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011794944/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829582.580890:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829582.580894:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829582.580897:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829582.580902:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4986 00000100:00080000:0.0:1787829582.580906:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787829582.580912:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011794816:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829582.580913:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829582.580924:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829582.580939:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011795072/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829582.580950:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829582.580954:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829582.580956:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829582.580961:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4986 00000100:00080000:0.0:1787829582.580965:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829582.580976:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829582.580982:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829582.580997:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011795200/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829582.581007:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829582.581072:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829582/real 1787829582] req@ffff907d90844780 x1874670011794816/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829582.581104:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011794816/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829582.581119:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829582.581124:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829582.581130:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011794816:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829582.581139:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011794944:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829582.581166:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829582.581182:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829582/real 1787829582] req@ffff907d90845680 x1874670011794944/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829582.581195:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011794944/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829582.581205:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829582.581209:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829582.581212:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011794944:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829582.581250:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011795072:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829582.581276:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829582.581291:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829582/real 1787829582] req@ffff907d90846d00 x1874670011795072/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829582.581304:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011795072/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829582.581316:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829582.581318:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829582.581321:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011795072:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829582.581327:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011795200:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829582.581348:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829582.581361:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829582/real 1787829582] req@ffff907d90845a40 x1874670011795200/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829582.581373:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011795200/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829582.581382:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829582.581385:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829582.581388:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011795200:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829582.708717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829582.708794:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829583.732747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829583.732831:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829584.756750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829584.756833:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829585.780741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829585.780770:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829586.804730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829586.804747:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829587.700760:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829587.700772:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829587.700777:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829587.700788:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4991 00000100:00080000:2.0:1787829587.700800:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829587.700808:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829587.700864:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011795328/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829587.700910:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829587.700916:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829587.700919:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829587.700925:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4991 00000100:00080000:2.0:1787829587.700930:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829587.700935:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829587.700947:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829587.700960:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011795456/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829587.700973:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011795328:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829587.700974:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829587.700981:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829587.700986:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829587.700996:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4991 00000100:00080000:2.0:1787829587.700999:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829587.701001:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829587.701005:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829587.701012:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011795584/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829587.701020:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829587.701024:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829587.701026:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829587.701032:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4991 00000100:00080000:2.0:1787829587.701036:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829587.701050:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829587.701057:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787829587.701062:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829587.701078:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011795712/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829587.701111:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829587/real 1787829587] req@ffff907d911483c0 x1874670011795328/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829587.701135:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011795328/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829587.701147:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829587.701153:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829587.701157:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011795328:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829587.701178:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011795456:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829587.701199:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829587.701208:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829587/real 1787829587] req@ffff907d9114bc00 x1874670011795456/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829587.701217:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011795456/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829587.701223:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829587.701225:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829587.701228:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011795456:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829587.701232:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011795584:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829587.701245:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829587.701260:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829587/real 1787829587] req@ffff907d91149680 x1874670011795584/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829587.701268:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011795584/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829587.701273:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829587.701275:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829587.701277:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011795584:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829587.701281:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011795712:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829587.701294:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829587.701300:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829587/real 1787829587] req@ffff907d9114b480 x1874670011795712/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829587.701307:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011795712/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829587.701312:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829587.701314:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829587.701316:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011795712:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829587.828751:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829587.828806:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829588.852716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829588.852783:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829589.876745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829589.876822:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829590.900717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829590.900762:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829591.924745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829591.924811:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829592.820728:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829592.820745:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829592.820752:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829592.820767:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 4996 00000100:00080000:2.0:1787829592.820783:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829592.820793:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829592.820839:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011795840/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829592.820878:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829592.820882:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829592.820885:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829592.820893:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4996 00000100:00080000:2.0:1787829592.820899:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829592.820904:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829592.820920:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829592.820941:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011795968/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829592.820961:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829592.820966:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829592.820974:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829592.820976:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011795840:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829592.820983:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4996 00000100:00080000:2.0:1787829592.820993:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829592.821000:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829592.821009:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829592.821030:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011796096/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829592.821041:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829592.821045:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829592.821049:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829592.821053:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 4996 00000100:00080000:2.0:1787829592.821058:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829592.821066:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787829592.821079:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829592.821090:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829592.821106:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011796224/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829592.821115:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829592/real 1787829592] req@ffff907d91149a40 x1874670011795840/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829592.821142:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011795840/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829592.821156:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829592.821161:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829592.821167:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011795840:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829592.821193:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011795968:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829592.821216:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829592.821228:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829592/real 1787829592] req@ffff907d9114ad00 x1874670011795968/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829592.821246:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011795968/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829592.821255:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829592.821258:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829592.821263:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011795968:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829592.821271:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011796096:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829592.821295:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829592.821318:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829592/real 1787829592] req@ffff907d9114a580 x1874670011796096/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829592.821332:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011796096/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829592.821345:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829592.821349:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829592.821357:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011796096:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829592.821366:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011796224:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829592.821386:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829592.821406:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829592/real 1787829592] req@ffff907d9114b840 x1874670011796224/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829592.821417:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011796224/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829592.821431:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829592.821434:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829592.821437:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011796224:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829592.948714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829592.948768:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829593.972744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829593.972820:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829594.996707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829594.996819:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829596.020728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829596.020769:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829597.044758:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829597.044836:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829597.940706:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829597.940713:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829597.940716:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829597.940724:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5001 00000100:00080000:2.0:1787829597.940732:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829597.940736:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829597.940762:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011796352/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829597.940781:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829597.940783:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829597.940784:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829597.940787:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5001 00000100:00080000:2.0:1787829597.940789:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829597.940791:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829597.940798:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829597.940807:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011796480/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829597.940818:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829597.940822:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787829597.940825:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011796352:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829597.940825:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829597.940836:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5001 00000100:00080000:2.0:1787829597.940840:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829597.940845:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829597.940850:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829597.940862:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011796608/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829597.940869:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829597.940871:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829597.940873:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829597.940875:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5001 00000100:00080000:2.0:1787829597.940877:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829597.940879:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:0.0:1787829597.940883:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:2.0:1787829597.940884:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829597.940893:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011796736/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829597.940914:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829597/real 1787829597] req@ffff907d911492c0 x1874670011796352/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829597.940927:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011796352/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829597.940934:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829597.940936:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829597.940939:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011796352:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829597.940954:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011796480:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829597.940966:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829597.940973:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829597/real 1787829597] req@ffff907d91148b40 x1874670011796480/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829597.940980:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011796480/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829597.940985:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829597.940987:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829597.940989:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011796480:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829597.940993:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011796608:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829597.941003:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829597.941008:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829597/real 1787829597] req@ffff907d9114a940 x1874670011796608/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829597.941014:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011796608/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829597.941018:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829597.941020:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829597.941022:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011796608:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829597.941025:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011796736:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829597.941033:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829597.941038:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829597/real 1787829597] req@ffff907d91148f00 x1874670011796736/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829597.941045:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011796736/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829597.941049:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829597.941050:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829597.941052:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011796736:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829598.068691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829598.068737:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829599.092709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829599.092769:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829600.116688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829600.116749:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829601.140676:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829601.140722:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829602.164714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829602.164739:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829603.060705:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829603.060713:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829603.060716:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829603.060724:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5006 00000100:00080000:3.0:1787829603.060734:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829603.060740:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829603.060775:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011796864/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829603.060804:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829603.060813:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829603.060815:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829603.060819:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5006 00000100:00080000:3.0:1787829603.060822:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829603.060827:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829603.060836:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829603.060849:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011796992/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829603.060850:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011796864:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829603.060863:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829603.060865:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829603.060867:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829603.060870:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5006 00000100:00080000:3.0:1787829603.060872:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829603.060874:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829603.060878:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829603.060886:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011797120/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829603.060891:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829603.060894:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829603.060895:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829603.060897:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5006 00000100:00080000:3.0:1787829603.060899:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829603.060901:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829603.060904:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787829603.060906:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829603.060913:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011797248/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829603.060939:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829603/real 1787829603] req@ffff907da562c3c0 x1874670011796864/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829603.060951:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011796864/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829603.060958:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829603.060961:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829603.060963:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011796864:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829603.060979:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011796992:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829603.060992:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829603.061006:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829603/real 1787829603] req@ffff907da562f0c0 x1874670011796992/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829603.061014:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011796992/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829603.061019:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829603.061021:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829603.061023:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011796992:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829603.061026:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011797120:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829603.061036:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829603.061042:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829603/real 1787829603] req@ffff907da562c000 x1874670011797120/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829603.061049:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011797120/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829603.061058:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829603.061060:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829603.061062:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011797120:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829603.061069:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011797248:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829603.061079:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829603.061084:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829603/real 1787829603] req@ffff907da562e940 x1874670011797248/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829603.061090:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011797248/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829603.061095:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829603.061096:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829603.061098:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011797248:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829603.188692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829603.188746:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829604.212701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829604.212725:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829605.236716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829605.236772:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829606.260681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829606.260761:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829607.284707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829607.284750:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829608.180722:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829608.180733:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829608.180738:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829608.180755:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5011 00000100:00080000:1.0:1787829608.180767:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829608.180774:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829608.180810:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011797376/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829608.180831:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829608.180835:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829608.180837:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829608.180842:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5011 00000100:00080000:1.0:1787829608.180846:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829608.180850:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829608.180862:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829608.180872:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011797376:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829608.180874:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011797504/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829608.180885:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829608.180888:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829608.180890:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829608.180893:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5011 00000100:00080000:1.0:1787829608.180896:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829608.180900:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829608.180904:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829608.180913:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011797632/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829608.180921:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829608.180925:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829608.180927:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829608.180930:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5011 00000100:00080000:1.0:1787829608.180933:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829608.180936:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829608.180940:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787829608.180942:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829608.180949:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011797760/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829608.180989:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829608/real 1787829608] req@ffff907d908a21c0 x1874670011797376/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829608.181007:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011797376/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829608.181014:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829608.181017:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829608.181021:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011797376:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829608.181046:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011797504:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829608.181065:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829608.181074:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829608/real 1787829608] req@ffff907d908a3840 x1874670011797504/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829608.181089:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011797504/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829608.181096:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829608.181098:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829608.181100:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011797504:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829608.181104:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011797632:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829608.181118:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829608.181125:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829608/real 1787829608] req@ffff907d908a1680 x1874670011797632/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829608.181132:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011797632/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829608.181137:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829608.181140:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829608.181144:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011797632:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829608.181149:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011797760:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829608.181168:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829608.181180:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829608/real 1787829608] req@ffff907d908a2580 x1874670011797760/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829608.181193:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011797760/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829608.181202:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829608.181206:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829608.181208:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011797760:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829608.308687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829608.308756:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829609.332710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829609.332767:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829610.356707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829610.356743:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829611.380757:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829611.380829:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787829612.404673:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829612.404727:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829613.300728:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829613.300739:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829613.300743:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829613.300754:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5017 00000100:00080000:1.0:1787829613.300770:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829613.300778:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829613.300812:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011797888/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829613.300837:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829613.300841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829613.300843:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829613.300847:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5017 00000100:00080000:1.0:1787829613.300851:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829613.300855:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829613.300865:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829613.300879:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011798016/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829613.300890:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011797888:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829613.300891:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829613.300900:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829613.300902:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829613.300906:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5017 00000100:00080000:1.0:1787829613.300910:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829613.300913:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829613.300918:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829613.300933:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011798144/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829613.300942:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829613.300945:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829613.300947:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829613.300950:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5017 00000800:00000100:0.0:1787829613.300955:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787829613.300956:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829613.300963:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829613.300976:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829613.300989:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011798272/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829613.301001:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829613/real 1787829613] req@ffff907d908a0f00 x1874670011797888/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829613.301017:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011797888/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829613.301026:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829613.301030:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829613.301033:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011797888:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829613.301051:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011798016:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829613.301068:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829613.301077:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829613/real 1787829613] req@ffff907d908a12c0 x1874670011798016/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829613.301085:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011798016/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829613.301092:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829613.301094:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829613.301103:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011798016:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829613.301108:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011798144:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829613.301124:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829613.301133:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829613/real 1787829613] req@ffff907d908a2580 x1874670011798144/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829613.301147:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011798144/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829613.301156:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829613.301160:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829613.301165:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011798144:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829613.301174:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011798272:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829613.301197:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829613.301211:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829613/real 1787829613] req@ffff907d908a1680 x1874670011798272/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829613.301220:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011798272/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829613.301226:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829613.301228:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829613.301231:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011798272:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829613.428721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829613.428742:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829614.452701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829614.452740:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829615.476729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829615.476776:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829616.500758:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829616.500793:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829617.524752:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829617.524831:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829618.420749:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829618.420759:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829618.420763:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829618.420773:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5022 00000100:00080000:2.0:1787829618.420784:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829618.420792:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829618.420830:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011798400/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829618.420860:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829618.420865:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829618.420867:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829618.420871:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5022 00000100:00080000:2.0:1787829618.420875:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829618.420879:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829618.420890:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829618.420905:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011798528/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829618.420929:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011798400:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829618.420942:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829618.420947:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829618.420950:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829618.420957:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5022 00000100:00080000:2.0:1787829618.420962:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829618.420967:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829618.420975:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829618.420991:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011798656/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829618.421004:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829618.421007:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829618.421010:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829618.421013:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5022 00000800:00000100:0.0:1787829618.421022:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787829618.421023:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829618.421032:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829618.421039:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829618.421053:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011798784/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829618.421088:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829618/real 1787829618] req@ffff907d91148f00 x1874670011798400/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829618.421108:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011798400/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829618.421117:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829618.421121:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829618.421124:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011798400:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829618.421149:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011798528:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829618.421169:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829618.421179:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829618/real 1787829618] req@ffff907d9114a940 x1874670011798528/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829618.421187:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011798528/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829618.421193:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829618.421196:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829618.421198:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011798528:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829618.421202:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011798656:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829618.421214:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829618.421241:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829618/real 1787829618] req@ffff907d91148b40 x1874670011798656/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829618.421250:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011798656/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829618.421255:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829618.421257:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829618.421260:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011798656:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829618.421264:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011798784:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829618.421278:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829618.421285:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829618/real 1787829618] req@ffff907d911492c0 x1874670011798784/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829618.421292:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011798784/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829618.421297:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829618.421299:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829618.421301:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011798784:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829618.548753:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829618.548784:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829619.572735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829619.572908:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829620.596758:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829620.596809:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829621.620704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829621.620756:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829622.644680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829622.644745:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829623.540705:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829623.540716:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829623.540720:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829623.540730:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5027 00000100:00080000:2.0:1787829623.540743:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829623.540750:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829623.540786:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011798912/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829623.540820:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829623.540826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829623.540827:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829623.540830:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5027 00000100:00080000:2.0:1787829623.540834:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829623.540837:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829623.540846:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829623.540858:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011799040/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829623.540866:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829623.540870:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787829623.540871:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011798912:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829623.540872:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829623.540882:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5027 00000100:00080000:2.0:1787829623.540886:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829623.540890:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829623.540895:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829623.540909:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011799168/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829623.540921:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829623.540924:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829623.540926:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829623.540929:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5027 00000100:00080000:2.0:1787829623.540932:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829623.540935:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829623.540939:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829623.540951:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011799296/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829623.540952:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829623.541002:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829623/real 1787829623] req@ffff907d9114b840 x1874670011798912/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829623.541023:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011798912/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829623.541032:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829623.541035:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829623.541038:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011798912:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829623.541056:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011799040:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829623.541076:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829623.541085:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829623/real 1787829623] req@ffff907d9114a580 x1874670011799040/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829623.541094:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011799040/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829623.541100:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829623.541102:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829623.541104:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011799040:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829623.541108:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011799168:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829623.541121:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829623.541127:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829623/real 1787829623] req@ffff907d9114ad00 x1874670011799168/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829623.541134:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011799168/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829623.541140:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829623.541142:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829623.541144:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011799168:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829623.541148:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011799296:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829623.541158:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829623.541163:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829623/real 1787829623] req@ffff907d91149a40 x1874670011799296/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829623.541171:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011799296/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829623.541176:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829623.541178:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829623.541179:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011799296:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829623.668703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829623.668727:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829624.692710:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829624.692767:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829625.716751:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829625.716789:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787829626.740756:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829626.740780:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829627.764701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829627.764730:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829628.660678:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829628.660686:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829628.660690:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829628.660699:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5032 00000100:00080000:2.0:1787829628.660708:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829628.660716:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829628.660756:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011799424/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829628.660782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829628.660786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829628.660788:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829628.660797:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5032 00000100:00080000:2.0:1787829628.660801:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829628.660804:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829628.660815:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829628.660828:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011799424:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829628.660836:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011799552/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829628.660855:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829628.660859:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829628.660862:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829628.660869:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5032 00000100:00080000:2.0:1787829628.660874:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829628.660879:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829628.660886:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829628.660908:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011799680/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829628.660914:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787829628.660934:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829628.660939:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829628.660943:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829628.660950:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5032 00000100:00080000:2.0:1787829628.660956:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829628.660963:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829628.660968:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829628/real 1787829628] req@ffff907d9114b480 x1874670011799424/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787829628.660974:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829628.660992:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011799424/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829628.660998:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011799808/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829628.661101:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829628.661107:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829628.661111:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011799424:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829628.661259:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011799552:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829628.661330:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829628.661351:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829628/real 1787829628] req@ffff907d91149680 x1874670011799552/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829628.661366:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011799552/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829628.661374:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829628.661378:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829628.661380:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011799552:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829628.661388:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011799680:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829628.661405:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829628.661412:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829628/real 1787829628] req@ffff907d9114bc00 x1874670011799680/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829628.661420:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011799680/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829628.661430:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829628.661433:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829628.661435:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011799680:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829628.661439:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011799808:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829628.661449:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829628.661455:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829628/real 1787829628] req@ffff907d911483c0 x1874670011799808/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829628.661462:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011799808/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829628.661467:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829628.661469:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829628.661471:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011799808:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829628.788773:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829628.788832:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829629.812680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829629.812723:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829630.836707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829630.836756:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829631.860681:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829631.860742:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829632.884713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829632.884754:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829633.780732:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829633.780742:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829633.780746:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829633.780757:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5037 00000100:00080000:2.0:1787829633.780768:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829633.780775:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829633.780812:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011799936/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829633.780836:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829633.780840:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829633.780842:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829633.780845:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5037 00000100:00080000:2.0:1787829633.780848:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829633.780851:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829633.780861:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829633.780872:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011800064/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829633.780883:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829633.780888:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1787829633.780888:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011799936:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829633.780893:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829633.780904:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5037 00000100:00080000:2.0:1787829633.780913:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829633.780917:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829633.780922:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829633.780934:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011800192/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829633.780943:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829633.780947:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829633.780948:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829633.780951:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5037 00000100:00080000:2.0:1787829633.780961:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829633.780964:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829633.780968:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787829633.780975:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829633.780981:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011800320/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829633.781041:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829633/real 1787829633] req@ffff907d911483c0 x1874670011799936/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829633.781066:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011799936/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829633.781076:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829633.781079:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829633.781083:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011799936:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829633.781106:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011800064:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829633.781151:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829633.781164:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829633/real 1787829633] req@ffff907d9114bc00 x1874670011800064/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829633.781175:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011800064/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829633.781182:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829633.781185:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829633.781188:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011800064:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829633.781193:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011800192:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829633.781208:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829633.781216:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829633/real 1787829633] req@ffff907d91149680 x1874670011800192/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829633.781226:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011800192/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829633.781238:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829633.781240:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829633.781243:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011800192:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829633.781247:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011800320:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829633.781259:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829633.781266:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829633/real 1787829633] req@ffff907d9114b480 x1874670011800320/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829633.781282:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011800320/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829633.781293:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829633.781297:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829633.781301:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011800320:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829633.908683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829633.908776:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829634.932715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829634.932770:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829635.956705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829635.956769:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829636.980730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829636.980756:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829638.004735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829638.004789:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829638.900782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829638.900793:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829638.900797:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829638.900809:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5042 00000100:00080000:3.0:1787829638.900821:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829638.900829:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829638.900873:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011800448/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829638.900905:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829638.900908:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829638.900909:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829638.900913:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5042 00000100:00080000:3.0:1787829638.900915:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829638.900918:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829638.900928:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829638.900941:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011800576/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829638.900949:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829638.900951:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829638.900952:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829638.900955:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5042 00000100:00080000:3.0:1787829638.900957:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829638.900960:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829638.900963:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829638.900970:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011800704/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829638.900977:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829638.900980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829638.900982:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829638.900987:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5042 00000100:00100000:0.0:1787829638.900990:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011800448:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829638.900990:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829638.900996:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829638.901003:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829638.901016:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011800832/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829638.901065:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829638.901129:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829638/real 1787829638] req@ffff907da562f840 x1874670011800448/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829638.901147:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011800448/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829638.901157:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829638.901160:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829638.901163:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011800448:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829638.901170:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011800576:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829638.901199:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829638.901213:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829638/real 1787829638] req@ffff907da562e1c0 x1874670011800576/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829638.901226:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011800576/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829638.901232:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829638.901235:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829638.901237:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011800576:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829638.901261:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011800704:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829638.901277:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829638.901284:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829638/real 1787829638] req@ffff907da562de00 x1874670011800704/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829638.901292:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011800704/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829638.901298:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829638.901301:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829638.901303:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011800704:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829638.901306:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011800832:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829638.901317:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829638.901323:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829638/real 1787829638] req@ffff907da562cb40 x1874670011800832/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829638.901335:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011800832/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829638.901341:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829638.901343:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829638.901344:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011800832:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829639.028708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829639.028777:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829640.052867:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829640.052903:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:0.0:1787829641.076788:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829641.076887:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829642.100753:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829642.100786:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829643.124707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829643.124753:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829644.020711:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829644.020724:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829644.020730:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829644.020744:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5047 00000100:00080000:3.0:1787829644.020759:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829644.020768:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829644.020815:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011800960/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829644.020849:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829644.020853:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829644.020855:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829644.020861:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5047 00000100:00080000:3.0:1787829644.020875:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829644.020882:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829644.020897:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829644.020912:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011800960:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829644.020918:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011801088/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829644.020934:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829644.020938:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829644.020941:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829644.020946:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5047 00000100:00080000:3.0:1787829644.020950:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829644.020955:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829644.020961:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829644.020975:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011801216/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829644.020986:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829644.020990:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829644.020993:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829644.020997:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5047 00000100:00080000:3.0:1787829644.021002:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829644.021006:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829644.021012:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829644.021036:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011801344/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829644.021074:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011801088:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829644.021127:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011801216:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829644.021159:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011801344:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829644.148677:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829644.148804:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829645.172688:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829645.172745:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829646.196731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829646.196788:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829647.092890:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787829647.092935:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829647.092953:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829647.093000:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829647.093011:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829647.093040:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:0.0:1787829647.093050:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829644/real 1787829647] req@ffff907da562cb40 x1874670011801088/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829647.093079:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011801088/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829647.093092:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829647.093097:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829647.093101:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011801088:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829647.093109:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829644/real 1787829647] req@ffff907da562de00 x1874670011801216/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829647.093131:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011801216/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829647.093140:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829647.093144:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829647.093148:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011801216:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829647.093154:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829644/real 1787829647] req@ffff907da562e1c0 x1874670011801344/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829647.093166:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011801344/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829647.093173:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829647.093177:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829647.093180:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011801344:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829647.093187:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829644/real 1787829647] req@ffff907da562da40 x1874670011800960/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829647.093198:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011800960/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829647.093218:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829647.093222:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829647.093226:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011800960:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829647.220711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829647.220758:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829648.244734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829648.244782:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829649.140679:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829649.140691:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829649.140696:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829649.140709:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5052 00000100:00080000:3.0:1787829649.140725:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829649.140736:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829649.140786:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011801472/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829649.140822:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829649.140826:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829649.140828:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829649.140836:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5052 00000100:00080000:3.0:1787829649.140842:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829649.140847:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829649.140859:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829649.140864:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011801472:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829649.140880:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011801600/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829649.140894:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829649.140898:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829649.140901:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829649.140907:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5052 00000100:00080000:3.0:1787829649.140914:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829649.140920:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829649.140927:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787829649.140941:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829649.140944:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011801728/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829649.140957:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829649.140961:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829649.140963:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829649.140968:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5052 00000100:00080000:3.0:1787829649.140973:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829649.140978:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829649.140983:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829649.140999:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011801856/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829649.141023:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829649/real 1787829649] req@ffff907da562f840 x1874670011801472/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829649.141053:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011801472/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829649.141067:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829649.141072:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829649.141077:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011801472:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829649.141105:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011801600:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829649.141144:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829649.141160:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829649/real 1787829649] req@ffff907da562e940 x1874670011801600/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829649.141173:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011801600/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829649.141183:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829649.141187:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829649.141191:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011801600:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829649.141199:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011801728:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829649.141225:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829649.141237:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829649/real 1787829649] req@ffff907da562c000 x1874670011801728/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829649.141251:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011801728/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829649.141257:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829649.141259:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829649.141262:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011801728:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829649.141267:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011801856:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829649.141281:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829649.141288:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829649/real 1787829649] req@ffff907da562f0c0 x1874670011801856/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829649.141296:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011801856/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829649.141303:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829649.141307:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829649.141311:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011801856:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829649.268719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829649.268753:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829650.292701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829650.292767:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829651.316705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829651.316835:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829652.340714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829652.340764:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829653.364684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829653.364759:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829654.260746:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829654.260760:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829654.260766:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829654.260779:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5058 00000100:00080000:3.0:1787829654.260793:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829654.260807:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829654.260857:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011801984/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829654.260886:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829654.260889:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829654.260890:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829654.260894:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5058 00000100:00080000:3.0:1787829654.260896:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829654.260899:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829654.260909:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829654.260919:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011802112/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829654.260928:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829654.260930:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829654.260932:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829654.260937:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5058 00000100:00100000:0.0:1787829654.260938:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011801984:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829654.260939:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829654.260946:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829654.260950:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829654.260959:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011802240/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829654.260965:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829654.260980:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829654.260982:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829654.260985:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5058 00000100:00080000:3.0:1787829654.260989:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829654.260992:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829654.260996:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829654.261007:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011802368/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829654.261031:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829654.261075:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829654/real 1787829654] req@ffff907da562c3c0 x1874670011801984/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829654.261090:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011801984/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829654.261104:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829654.261107:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829654.261111:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011801984:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829654.261135:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011802112:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829654.261151:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829654.261159:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829654/real 1787829654] req@ffff907da562fc00 x1874670011802112/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829654.261167:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011802112/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829654.261173:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829654.261175:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829654.261177:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011802112:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829654.261181:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011802240:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829654.261192:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829654.261199:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829654/real 1787829654] req@ffff907da562d2c0 x1874670011802240/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829654.261211:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011802240/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829654.261221:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829654.261224:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829654.261228:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011802240:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829654.261235:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011802368:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829654.261256:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829654.261268:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829654/real 1787829654] req@ffff907da562e580 x1874670011802368/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829654.261280:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011802368/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829654.261288:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829654.261292:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829654.261295:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011802368:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829654.388737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829654.388789:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829655.412724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829655.412789:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829656.436739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829656.436753:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829657.460729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829657.460783:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829658.484730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829658.484827:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829659.380738:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829659.380750:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829659.380755:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829659.380775:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5063 00000100:00080000:3.0:1787829659.380787:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829659.380795:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829659.380835:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011802496/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829659.380865:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829659.380869:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829659.380872:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829659.380876:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5063 00000100:00080000:3.0:1787829659.380881:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829659.380886:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829659.380897:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829659.380908:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011802496:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829659.380916:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011802624/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829659.380931:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829659.380949:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829659.380952:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829659.380957:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5063 00000100:00080000:3.0:1787829659.380963:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829659.380968:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829659.380974:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787829659.380984:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829659.380991:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011802752/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829659.381004:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829659.381008:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829659.381011:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829659.381015:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5063 00000100:00080000:3.0:1787829659.381020:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829659.381025:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829659.381028:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829659/real 1787829659] req@ffff907da562cf00 x1874670011802496/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787829659.381033:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829659.381051:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011802496/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829659.381056:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011802880/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829659.381064:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829659.381071:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829659.381077:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011802496:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829659.381101:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011802624:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829659.381118:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829659.381127:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829659/real 1787829659] req@ffff907da562c780 x1874670011802624/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829659.381135:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011802624/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829659.381141:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829659.381143:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829659.381145:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011802624:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829659.381149:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011802752:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829659.381160:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829659.381166:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829659/real 1787829659] req@ffff907da562ed00 x1874670011802752/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829659.381174:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011802752/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829659.381179:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829659.381181:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829659.381182:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011802752:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829659.381186:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011802880:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829659.381195:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829659.381201:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829659/real 1787829659] req@ffff907da562f480 x1874670011802880/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829659.381207:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011802880/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829659.381217:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829659.381219:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829659.381222:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011802880:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829659.508749:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829659.508896:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829660.532706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829660.532815:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829661.556731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829661.556779:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829662.580714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829662.580804:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829663.604671:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829663.604799:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829664.500749:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829664.500759:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829664.500764:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829664.500776:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5068 00000100:00080000:2.0:1787829664.500788:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829664.500795:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829664.500839:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011803008/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829664.500870:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829664.500873:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829664.500874:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829664.500878:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5068 00000100:00080000:2.0:1787829664.500882:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829664.500887:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829664.500910:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829664.500914:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011803008:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829664.500929:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011803136/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829664.500944:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829664.500948:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829664.500951:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829664.500954:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5068 00000100:00080000:2.0:1787829664.500976:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829664.500981:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829664.500988:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787829664.500988:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829664.501004:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011803264/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829664.501013:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829664.501016:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829664.501018:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829664.501022:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5068 00000100:00080000:2.0:1787829664.501025:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829664.501028:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829664.501033:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829664.501036:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829664/real 1787829664] req@ffff907d91149a40 x1874670011803008/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829664.501044:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011803392/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829664.501056:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011803008/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829664.501066:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829664.501132:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829664.501135:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011803008:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829664.501156:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011803136:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829664.501291:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829664.501346:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829664/real 1787829664] req@ffff907d9114ad00 x1874670011803136/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829664.501366:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011803136/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829664.501372:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829664.501375:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829664.501377:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011803136:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829664.501382:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011803264:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829664.501401:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829664.501409:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829664/real 1787829664] req@ffff907d9114a580 x1874670011803264/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829664.501418:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011803264/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829664.501424:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829664.501426:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829664.501428:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011803264:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829664.501432:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011803392:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829664.501442:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829664.501448:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829664/real 1787829664] req@ffff907d9114b840 x1874670011803392/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829664.501455:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011803392/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829664.501461:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829664.501463:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829664.501464:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011803392:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829664.628709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829664.628782:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829665.652723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829665.652822:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829666.676708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829666.676896:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829667.700713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829667.700777:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829668.724733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829668.724796:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829669.620712:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829669.620723:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829669.620727:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829669.620738:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5073 00000100:00080000:3.0:1787829669.620750:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829669.620757:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829669.620799:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011803520/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829669.620829:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829669.620832:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829669.620833:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829669.620837:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5073 00000100:00080000:3.0:1787829669.620839:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829669.620841:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829669.620851:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829669.620875:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011803648/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829669.620885:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829669.620889:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011803520:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829669.620891:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829669.620896:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829669.620902:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5073 00000100:00080000:3.0:1787829669.620910:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829669.620914:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829669.620920:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829669.620935:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011803776/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829669.620950:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829669.620954:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829669.620956:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829669.620959:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5073 00000100:00080000:3.0:1787829669.620963:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829669.620969:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:0.0:1787829669.620971:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:3.0:1787829669.620978:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829669.620993:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011803904/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829669.621030:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829669/real 1787829669] req@ffff907da562d680 x1874670011803520/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829669.621049:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011803520/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829669.621058:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829669.621061:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829669.621064:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011803520:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829669.621093:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011803648:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829669.621120:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829669.621131:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829669/real 1787829669] req@ffff907da562f480 x1874670011803648/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829669.621145:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011803648/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829669.621152:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829669.621155:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829669.621159:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011803648:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829669.621166:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011803776:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829669.621189:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829669.621208:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829669/real 1787829669] req@ffff907da562ed00 x1874670011803776/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829669.621217:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011803776/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829669.621223:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829669.621226:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829669.621228:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011803776:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829669.621232:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011803904:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829669.621247:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829669.621254:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829669/real 1787829669] req@ffff907da562c780 x1874670011803904/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829669.621264:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011803904/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829669.621273:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829669.621277:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829669.621280:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011803904:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829669.748738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829669.748833:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829670.772756:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829670.772891:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829671.796721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829671.796813:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829672.820667:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829672.820750:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829673.844767:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829673.844845:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829674.740759:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829674.740771:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829674.740775:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829674.740786:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5078 00000100:00080000:2.0:1787829674.740800:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829674.740807:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829674.740860:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011804032/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829674.740887:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829674.740889:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829674.740890:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829674.740894:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5078 00000100:00080000:2.0:1787829674.740896:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829674.740899:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829674.740909:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829674.740919:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011804160/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829674.740927:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829674.740929:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829674.740931:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829674.740933:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5078 00000100:00080000:2.0:1787829674.740935:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829674.740938:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829674.740941:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829674.740950:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011804032:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829674.740950:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011804288/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829674.740966:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829674.740969:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829674.740971:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829674.740973:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5078 00000100:00080000:2.0:1787829674.740974:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829674.740976:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829674.740979:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829674.740987:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011804416/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829674.741042:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829674.741114:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829674/real 1787829674] req@ffff907d911492c0 x1874670011804032/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829674.741137:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011804032/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829674.741151:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829674.741156:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829674.741161:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011804032:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829674.741169:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011804160:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829674.741196:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829674.741211:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829674/real 1787829674] req@ffff907d91148b40 x1874670011804160/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829674.741225:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011804160/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829674.741234:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829674.741238:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829674.741242:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011804160:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829674.741271:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011804288:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829674.741296:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829674.741308:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829674/real 1787829674] req@ffff907d9114a940 x1874670011804288/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829674.741322:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011804288/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829674.741339:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829674.741342:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829674.741347:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011804288:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829674.741353:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011804416:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829674.741375:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829674.741387:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829674/real 1787829674] req@ffff907d91148f00 x1874670011804416/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829674.741400:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011804416/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829674.741409:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829674.741412:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829674.741416:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011804416:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787829674.868804:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829674.868870:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787829675.892731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829675.892801:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829676.916741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829676.916826:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829677.940741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829677.940817:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829678.964742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829678.964837:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829679.860732:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829679.860743:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829679.860747:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829679.860759:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5083 00000100:00080000:2.0:1787829679.860771:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829679.860779:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829679.860839:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011804544/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829679.860868:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829679.860871:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829679.860873:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829679.860876:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5083 00000100:00080000:2.0:1787829679.860880:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829679.860883:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829679.860894:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829679.860913:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011804672/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829679.860922:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011804544:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829679.860930:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829679.860935:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829679.860937:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829679.860942:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5083 00000100:00080000:2.0:1787829679.860946:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829679.860950:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829679.860955:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829679.860965:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011804800/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829679.860973:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829679.860976:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829679.860978:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829679.860988:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5083 00000100:00080000:2.0:1787829679.860995:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829679.861000:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829679.861006:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787829679.861022:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829679.861023:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011804928/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829679.861074:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829679/real 1787829679] req@ffff907d91148f00 x1874670011804544/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829679.861090:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011804544/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829679.861100:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829679.861104:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829679.861107:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011804544:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829679.861130:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011804672:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829679.861148:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829679.861158:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829679/real 1787829679] req@ffff907d9114a940 x1874670011804672/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829679.861166:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011804672/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829679.861172:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829679.861175:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829679.861177:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011804672:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829679.861181:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011804800:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829679.861192:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829679.861199:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829679/real 1787829679] req@ffff907d91148b40 x1874670011804800/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829679.861207:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011804800/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829679.861212:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829679.861214:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829679.861216:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011804800:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829679.861219:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011804928:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829679.861229:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829679.861235:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829679/real 1787829679] req@ffff907d911492c0 x1874670011804928/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829679.861243:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011804928/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829679.861248:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829679.861250:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829679.861252:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011804928:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829679.988703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829679.988767:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829681.012745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829681.012817:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829682.036711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829682.036786:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829683.060716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829683.060853:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829684.084691:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829684.084808:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829684.980722:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829684.980730:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829684.980734:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829684.980743:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5088 00000100:00080000:0.0:1787829684.980752:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829684.980768:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829684.980798:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011805056/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829684.980828:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829684.980832:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829684.980835:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829684.980840:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5088 00000100:00080000:0.0:1787829684.980845:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829684.980851:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829684.980861:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829684.980862:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011805056:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829684.980876:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011805184/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829684.980891:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829684.980894:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829684.980896:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829684.980899:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5088 00000100:00080000:0.0:1787829684.980903:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829684.980905:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829684.980909:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1787829684.980917:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829684.980919:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011805312/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829684.980928:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829684.980931:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829684.980933:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829684.980936:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5088 00000100:00080000:0.0:1787829684.980938:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829684.980940:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829684.980944:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829684.980952:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011805440/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829684.980966:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829684/real 1787829684] req@ffff907d90847480 x1874670011805056/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829684.980987:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011805056/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829684.980995:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829684.980998:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829684.981001:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011805056:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829684.981019:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011805184:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829684.981034:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829684.981061:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829684/real 1787829684] req@ffff907d90846940 x1874670011805184/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829684.981070:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011805184/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829684.981077:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829684.981079:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829684.981081:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011805184:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829684.981085:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011805312:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829684.981102:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829684.981113:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829684/real 1787829684] req@ffff907d90844f00 x1874670011805312/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829684.981127:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011805312/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829684.981145:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829684.981159:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829684.981162:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011805312:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829684.981167:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011805440:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829684.981185:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829684.981209:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829684/real 1787829684] req@ffff907d90845a40 x1874670011805440/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829684.981221:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011805440/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829684.981239:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829684.981243:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829684.981246:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011805440:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829685.108702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829685.108790:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829686.132666:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829686.132763:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829687.156694:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829687.156763:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829688.180680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829688.180761:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829689.204741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829689.204831:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829690.100830:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829690.100845:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829690.100851:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829690.100865:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5093 00000100:00080000:2.0:1787829690.100879:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829690.100889:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829690.100933:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011805568/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829690.100972:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829690.100978:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829690.100981:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829690.100988:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5093 00000100:00080000:2.0:1787829690.101012:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829690.101020:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829690.101033:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011805568:192.168.203.104@tcp:250:kworker.0 00000008:00100000:2.0:1787829690.101038:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829690.101077:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011805696/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829690.101101:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829690.101107:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829690.101111:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829690.101118:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5093 00000100:00080000:2.0:1787829690.101126:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787829690.101136:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787829690.101138:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829690.101151:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829690.101176:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011805824/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829690.101191:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829690/real 1787829690] req@ffff907d9114b840 x1874670011805568/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829690.101197:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829690.101217:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829690.101226:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829690.101236:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5093 00000100:00080000:2.0:1787829690.101244:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787829690.101245:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011805568/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829690.101260:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00080000:3.0:1787829690.101287:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1787829690.101292:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1787829690.101294:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829690.101307:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011805568:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829690.101315:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011805952/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829690.101359:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011805696:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829690.101392:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829690.101405:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829690/real 1787829690] req@ffff907d9114a580 x1874670011805696/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829690.101414:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011805696/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829690.101420:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829690.101423:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829690.101425:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011805696:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829690.101430:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011805824:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829690.101444:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829690.101451:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829690/real 1787829690] req@ffff907d9114ad00 x1874670011805824/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829690.101458:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011805824/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829690.101464:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829690.101466:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829690.101467:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011805824:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829690.101472:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011805952:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829690.101483:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829690.101498:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829690/real 1787829690] req@ffff907d91149a40 x1874670011805952/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829690.101518:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011805952/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829690.101524:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829690.101526:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829690.101528:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011805952:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829690.228805:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829690.228826:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829691.252701:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829691.252808:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829692.276850:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829692.276890:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829693.300741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829693.300815:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787829694.324833:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829694.324905:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:2.0:1787829695.220761:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829695.220773:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829695.220777:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829695.220788:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5098 00000100:00080000:2.0:1787829695.220801:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829695.220809:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829695.220851:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011806080/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829695.220880:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829695.220883:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829695.220884:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829695.220888:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5098 00000100:00080000:2.0:1787829695.220891:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829695.220893:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829695.220903:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829695.220916:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011806208/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829695.220925:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829695.220927:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829695.220929:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829695.220934:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5098 00000100:00100000:3.0:1787829695.220935:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011806080:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829695.220936:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829695.220943:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829695.220947:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829695.220956:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011806336/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829695.220964:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829695.220967:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829695.220969:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829695.220971:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5098 00000100:00080000:2.0:1787829695.220974:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829695.220976:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829695.220979:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829695.220986:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011806464/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829695.221010:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829695.221063:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829695/real 1787829695] req@ffff907d9114b480 x1874670011806080/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829695.221078:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011806080/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829695.221087:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829695.221090:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829695.221093:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011806080:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829695.221124:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011806208:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829695.221144:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829695.221159:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829695/real 1787829695] req@ffff907d91149680 x1874670011806208/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829695.221173:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011806208/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829695.221181:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829695.221185:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829695.221188:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011806208:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829695.221208:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011806336:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829695.221233:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829695.221246:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829695/real 1787829695] req@ffff907d9114bc00 x1874670011806336/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829695.221259:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011806336/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829695.221267:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829695.221271:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829695.221275:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011806336:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829695.221282:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011806464:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829695.221301:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829695.221313:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829695/real 1787829695] req@ffff907d911483c0 x1874670011806464/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829695.221327:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011806464/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829695.221335:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829695.221339:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829695.221343:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011806464:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829695.348732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829695.348784:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829696.372747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829696.372815:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829697.396735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829697.396793:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829698.420762:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829698.420841:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829699.444727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829699.444780:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829700.340707:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829700.340719:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829700.340724:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829700.340736:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5104 00000100:00080000:2.0:1787829700.340749:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829700.340757:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829700.340802:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011806592/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829700.340831:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829700.340834:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829700.340837:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829700.340841:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5104 00000100:00080000:2.0:1787829700.340845:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829700.340850:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829700.340863:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829700.340878:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011806720/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829700.340886:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011806592:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829700.340895:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829700.340899:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829700.340901:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829700.340908:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5104 00000100:00080000:2.0:1787829700.340914:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829700.340920:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829700.340926:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829700.340941:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011806848/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829700.340949:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829700.340955:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829700.340958:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829700.340964:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5104 00000800:00000100:0.0:1787829700.340970:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787829700.340972:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829700.340981:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829700.340991:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829700.341013:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011806976/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829700.341019:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829700/real 1787829700] req@ffff907d911483c0 x1874670011806592/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829700.341080:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011806592/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829700.341095:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829700.341118:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829700.341123:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011806592:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829700.341147:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011806720:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829700.341168:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829700.341182:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829700/real 1787829700] req@ffff907d9114bc00 x1874670011806720/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829700.341207:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011806720/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829700.341218:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829700.341221:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829700.341226:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011806720:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829700.341232:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011806848:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829700.341255:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829700.341283:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829700/real 1787829700] req@ffff907d91149680 x1874670011806848/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829700.341305:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011806848/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829700.341329:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829700.341335:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829700.341341:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011806848:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829700.341355:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011806976:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829700.341409:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829700.341430:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829700/real 1787829700] req@ffff907d9114b480 x1874670011806976/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829700.341443:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011806976/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829700.341449:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829700.341452:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829700.341454:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011806976:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829700.468729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829700.468784:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829701.492697:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829701.492752:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829702.516725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829702.516776:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829703.540706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829703.540823:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829704.564716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829704.564775:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829705.460846:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829705.460860:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829705.460865:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829705.460879:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5109 00000100:00080000:0.0:1787829705.460892:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829705.460902:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829705.460951:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011807104/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829705.460988:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829705.460993:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829705.460995:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829705.461000:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5109 00000100:00080000:0.0:1787829705.461004:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829705.461009:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829705.461022:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829705.461039:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011807104:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829705.461040:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011807232/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829705.461059:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829705.461063:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829705.461065:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829705.461070:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5109 00000100:00080000:0.0:1787829705.461075:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829705.461079:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829705.461086:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829705.461099:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011807360/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829705.461111:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829705.461115:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829705.461120:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829705.461125:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5109 00000800:00000100:3.0:1787829705.461127:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787829705.461130:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829705.461136:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829705.461143:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829705.461156:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011807488/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829705.461185:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829705/real 1787829705] req@ffff907d90846d00 x1874670011807104/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829705.461203:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011807104/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829705.461214:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829705.461218:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829705.461221:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011807104:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829705.461244:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011807232:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829705.461260:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829705.461270:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829705/real 1787829705] req@ffff907d90845680 x1874670011807232/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829705.461278:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011807232/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829705.461284:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829705.461287:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829705.461290:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011807232:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829705.461296:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011807360:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829705.461318:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829705.461328:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829705/real 1787829705] req@ffff907d90844780 x1874670011807360/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829705.461340:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011807360/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829705.461348:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829705.461351:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829705.461354:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011807360:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829705.461361:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011807488:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829705.461379:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829705.461387:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829705/real 1787829705] req@ffff907d908452c0 x1874670011807488/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829705.461396:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011807488/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829705.461401:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829705.461403:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829705.461405:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011807488:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829705.588745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829705.588790:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829706.612750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829706.612805:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829707.636719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829707.636796:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829708.660705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829708.660788:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829709.684682:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829709.684752:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829710.580720:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829710.580735:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829710.580740:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829710.580752:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5114 00000100:00080000:2.0:1787829710.580765:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829710.580772:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829710.580820:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011807616/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829710.580853:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829710.580857:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829710.580858:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829710.580862:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5114 00000100:00080000:2.0:1787829710.580865:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829710.580869:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829710.580880:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829710.580896:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011807616:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829710.580897:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011807744/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829710.580917:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829710.580918:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829710.580919:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829710.580923:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5114 00000100:00080000:2.0:1787829710.580925:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829710.580928:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829710.580932:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829710.580943:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011807872/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829710.580949:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829710.580952:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829710.580954:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829710.580956:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5114 00000100:00080000:2.0:1787829710.580966:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829710.580968:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829710.580972:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829710.580979:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011808000/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829710.581016:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011807744:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829710.581041:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011807872:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829710.581063:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011808000:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829710.708759:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829710.708815:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829711.732746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829711.732825:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829712.756718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829712.756773:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829713.652914:0:2337:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787829713.652969:0:2337:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829713.652987:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829713.653039:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829713.653050:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829713.653064:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:3.0:1787829713.653092:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829710/real 1787829713] req@ffff907d9114ad00 x1874670011807744/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829713.653122:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011807744/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829713.653136:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829713.653143:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829713.653147:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011807744:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829713.653157:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829710/real 1787829713] req@ffff907d9114a580 x1874670011807872/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829713.653171:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011807872/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829713.653181:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829713.653185:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829713.653190:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011807872:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829713.653198:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829710/real 1787829713] req@ffff907d9114b840 x1874670011808000/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829713.653213:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011808000/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829713.653222:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829713.653226:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829713.653230:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011808000:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829713.653238:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829710/real 1787829713] req@ffff907d91149a40 x1874670011807616/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829713.653253:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011807616/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829713.653262:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829713.653267:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829713.653270:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011807616:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829713.780699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829713.780779:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829714.804747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829714.804797:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829715.700690:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829715.700701:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829715.700705:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829715.700716:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5119 00000100:00080000:3.0:1787829715.700729:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829715.700737:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829715.700777:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011808128/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829715.700803:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829715.700806:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829715.700808:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829715.700811:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5119 00000100:00080000:3.0:1787829715.700813:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829715.700816:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829715.700825:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829715.700834:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011808256/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829715.700841:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829715.700843:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829715.700844:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829715.700848:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5119 00000100:00080000:3.0:1787829715.700851:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829715.700854:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829715.700858:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829715.700866:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011808384/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829715.700874:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829715.700878:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829715.700882:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829715.700884:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011808128:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829715.700891:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5119 00000100:00080000:3.0:1787829715.700897:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829715.700902:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829715.700908:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829715.700922:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011808512/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829715.700976:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829715.701060:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829715/real 1787829715] req@ffff907da562cf00 x1874670011808128/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829715.701083:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011808128/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829715.701096:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829715.701102:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829715.701107:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011808128:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829715.701116:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011808256:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829715.701146:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829715.701168:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829715/real 1787829715] req@ffff907da562e580 x1874670011808256/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829715.701183:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011808256/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829715.701193:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829715.701196:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829715.701200:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011808256:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829715.701231:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011808384:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829715.701257:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829715.701270:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829715/real 1787829715] req@ffff907da562d2c0 x1874670011808384/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829715.701284:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011808384/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829715.701293:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829715.701296:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829715.701300:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011808384:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829715.701306:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011808512:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829715.701328:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829715.701341:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829715/real 1787829715] req@ffff907da562fc00 x1874670011808512/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829715.701354:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011808512/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829715.701363:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829715.701366:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829715.701370:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011808512:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829715.828713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829715.828753:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829716.852696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829716.852773:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829717.876719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829717.876822:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829718.900709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829718.900768:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829719.924741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829719.924779:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829720.820693:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829720.820701:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829720.820705:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829720.820712:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5124 00000100:00080000:2.0:1787829720.820722:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829720.820727:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829720.820755:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011808640/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829720.820774:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829720.820777:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829720.820778:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829720.820781:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5124 00000100:00080000:2.0:1787829720.820783:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829720.820785:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829720.820794:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829720.820804:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011808768/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829720.820812:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829720.820816:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787829720.820818:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011808640:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829720.820818:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829720.820827:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5124 00000100:00080000:2.0:1787829720.820830:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829720.820833:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829720.820856:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829720.820865:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011808896/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829720.820871:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829720.820874:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829720.820875:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829720.820879:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5124 00000800:00000100:0.0:1787829720.820881:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787829720.820883:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829720.820888:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829720.820891:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829720.820900:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011809024/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829720.820919:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829720/real 1787829720] req@ffff907d911492c0 x1874670011808640/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829720.820941:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011808640/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829720.820949:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829720.820951:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829720.820955:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011808640:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829720.820978:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011808768:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829720.821002:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829720.821016:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829720/real 1787829720] req@ffff907d91148b40 x1874670011808768/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829720.821025:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011808768/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829720.821031:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829720.821033:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829720.821035:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011808768:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829720.821039:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011808896:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829720.821054:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829720.821061:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829720/real 1787829720] req@ffff907d9114a940 x1874670011808896/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829720.821069:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011808896/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829720.821078:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829720.821080:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829720.821082:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011808896:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829720.821086:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011809024:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829720.821095:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829720.821100:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829720/real 1787829720] req@ffff907d91148f00 x1874670011809024/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829720.821107:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011809024/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829720.821112:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829720.821115:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829720.821116:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011809024:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829720.948722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829720.948867:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829721.972744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829721.972803:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829722.996685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829722.996737:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829724.020736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829724.020814:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829725.044677:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829725.044737:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829725.940705:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829725.940716:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829725.940721:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829725.940732:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5129 00000100:00080000:3.0:1787829725.940745:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829725.940752:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829725.940796:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011809152/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829725.940824:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829725.940827:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829725.940829:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829725.940832:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5129 00000100:00080000:3.0:1787829725.940835:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829725.940837:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829725.940847:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829725.940856:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011809280/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829725.940863:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829725.940866:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829725.940868:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829725.940870:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5129 00000100:00080000:3.0:1787829725.940873:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829725.940875:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829725.940878:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829725.940887:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011809408/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829725.940889:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011809152:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829725.940903:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829725.940906:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829725.940907:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829725.940909:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5129 00000100:00080000:3.0:1787829725.940910:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829725.940912:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829725.940916:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829725.940926:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011809536/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829725.940976:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829725.941035:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829725/real 1787829725] req@ffff907da562c3c0 x1874670011809152/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829725.941060:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011809152/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829725.941073:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829725.941078:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829725.941083:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011809152:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829725.941093:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011809280:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829725.941119:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829725.941133:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829725/real 1787829725] req@ffff907da562f0c0 x1874670011809280/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829725.941146:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011809280/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829725.941154:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829725.941158:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829725.941161:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011809280:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829725.941192:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011809408:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829725.941213:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829725.941224:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829725/real 1787829725] req@ffff907da562c000 x1874670011809408/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829725.941237:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011809408/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829725.941246:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829725.941250:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829725.941253:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011809408:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829725.941260:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011809536:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829725.941279:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829725.941290:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829725/real 1787829725] req@ffff907da562e940 x1874670011809536/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829725.941303:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011809536/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829725.941321:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829725.941326:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829725.941329:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011809536:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829726.068744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829726.068814:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829727.092685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829727.092767:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829728.116715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829728.116751:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829729.140678:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829729.140738:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829730.164744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829730.164823:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829731.060731:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829731.060743:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829731.060752:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829731.060764:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5134 00000100:00080000:3.0:1787829731.060776:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829731.060783:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829731.060830:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011809664/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829731.060863:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829731.060868:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829731.060871:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829731.060876:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5134 00000100:00080000:3.0:1787829731.060881:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829731.060886:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829731.060897:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829731.060912:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011809664:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829731.060912:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011809792/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829731.060927:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829731.060930:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829731.060931:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829731.060933:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5134 00000100:00080000:3.0:1787829731.060936:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829731.060940:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829731.060943:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829731.060952:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011809920/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829731.060960:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829731.060964:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829731.060966:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829731.060970:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5134 00000100:00080000:3.0:1787829731.060984:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829731.060990:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787829731.060991:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829731.061000:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829731.061016:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011810048/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829731.061047:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829731/real 1787829731] req@ffff907da562f840 x1874670011809664/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829731.061064:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011809664/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829731.061074:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829731.061078:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829731.061081:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011809664:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829731.061104:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011809792:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829731.061120:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829731.061128:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829731/real 1787829731] req@ffff907da562da40 x1874670011809792/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829731.061136:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011809792/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829731.061147:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829731.061149:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829731.061151:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011809792:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829731.061155:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011809920:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829731.061166:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829731.061172:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829731/real 1787829731] req@ffff907da562e1c0 x1874670011809920/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829731.061183:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011809920/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829731.061192:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829731.061196:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829731.061199:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011809920:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829731.061206:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011810048:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829731.061227:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829731.061238:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829731/real 1787829731] req@ffff907da562de00 x1874670011810048/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829731.061246:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011810048/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829731.061252:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829731.061255:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829731.061257:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011810048:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829731.188750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829731.188882:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829732.212763:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829732.212818:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829733.236727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829733.236779:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829734.260712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829734.260772:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829735.284762:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829735.284827:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829736.180687:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829736.180696:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829736.180700:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829736.180709:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5139 00000100:00080000:3.0:1787829736.180718:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829736.180725:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829736.180754:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011810176/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829736.180782:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829736.180785:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829736.180787:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829736.180792:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5139 00000100:00080000:3.0:1787829736.180794:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829736.180798:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829736.180806:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829736.180815:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011810176:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829736.180820:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011810304/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829736.180830:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829736.180833:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829736.180835:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829736.180839:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5139 00000100:00080000:3.0:1787829736.180842:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829736.180846:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829736.180852:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829736.180865:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011810432/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829736.180874:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787829736.180883:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829736.180889:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829736.180892:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829736.180898:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5139 00000100:00080000:3.0:1787829736.180904:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829736.180909:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829736.180915:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829736.180920:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829736/real 1787829736] req@ffff907da562cb40 x1874670011810176/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829736.180935:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011810560/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829736.180942:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011810176/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829736.180951:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829736.180954:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829736.180958:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011810176:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829736.180979:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011810304:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829736.180998:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829736.181010:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829736/real 1787829736] req@ffff907da562de00 x1874670011810304/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829736.181020:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011810304/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829736.181026:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829736.181028:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829736.181030:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011810304:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829736.181034:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011810432:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829736.181046:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829736.181052:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829736/real 1787829736] req@ffff907da562e1c0 x1874670011810432/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829736.181060:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011810432/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829736.181070:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829736.181072:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829736.181076:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011810432:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829736.181181:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011810560:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829736.181207:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829736.181225:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829736/real 1787829736] req@ffff907da562da40 x1874670011810560/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829736.181238:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011810560/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829736.181246:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829736.181249:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829736.181251:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011810560:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829736.308734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829736.308753:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829737.332740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829737.332817:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829738.356713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829738.356764:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829739.380713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829739.380774:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829740.404669:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829740.404726:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829741.300723:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829741.300731:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829741.300735:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829741.300743:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5145 00000100:00080000:2.0:1787829741.300753:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829741.300767:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829741.300806:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011810688/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829741.300834:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829741.300839:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829741.300841:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829741.300846:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5145 00000100:00080000:2.0:1787829741.300851:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829741.300856:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829741.300866:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829741.300870:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011810688:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829741.300883:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011810816/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829741.300892:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829741.300895:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829741.300897:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829741.300900:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5145 00000100:00080000:2.0:1787829741.300903:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829741.300906:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829741.300909:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829741.300917:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011810944/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829741.300924:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829741.300927:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829741.300928:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787829741.300934:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787829741.300935:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5145 00000100:00080000:2.0:1787829741.300941:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829741.300944:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829741.300948:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829741.300956:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011811072/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829741.300993:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829741/real 1787829741] req@ffff907d91148f00 x1874670011810688/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829741.301008:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011810688/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829741.301015:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829741.301018:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829741.301021:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011810688:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829741.301049:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011810816:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829741.301076:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829741.301090:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829741/real 1787829741] req@ffff907d9114a940 x1874670011810816/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829741.301100:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011810816/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829741.301106:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829741.301108:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829741.301110:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011810816:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829741.301115:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011810944:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829741.301131:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829741.301139:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829741/real 1787829741] req@ffff907d91148b40 x1874670011810944/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829741.301147:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011810944/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829741.301152:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829741.301155:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829741.301157:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011810944:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829741.301160:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011811072:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829741.301171:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829741.301177:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829741/real 1787829741] req@ffff907d911492c0 x1874670011811072/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829741.301184:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011811072/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829741.301189:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829741.301191:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829741.301193:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011811072:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829741.428708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829741.428775:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829742.452722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829742.452747:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829743.476731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829743.476827:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829744.500703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829744.500769:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829745.524750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829745.524815:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829746.420689:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829746.420697:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829746.420701:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829746.420710:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5150 00000100:00080000:2.0:1787829746.420719:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829746.420724:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829746.420757:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011811200/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829746.420781:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829746.420784:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829746.420785:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829746.420788:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5150 00000100:00080000:2.0:1787829746.420790:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829746.420792:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829746.420800:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829746.420810:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011811328/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829746.420818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829746.420821:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829746.420822:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829746.420826:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5150 00000100:00080000:2.0:1787829746.420828:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829746.420830:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829746.420834:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829746.420844:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011811456/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829746.420855:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829746.420856:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011811200:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829746.420870:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829746.420873:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829746.420879:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5150 00000100:00080000:2.0:1787829746.420883:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829746.420888:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829746.420896:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829746.420914:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011811584/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829746.420956:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829746.421004:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829746/real 1787829746] req@ffff907d91149a40 x1874670011811200/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829746.421020:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011811200/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829746.421031:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829746.421035:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829746.421039:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011811200:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829746.421045:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011811328:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829746.421063:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829746.421081:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829746/real 1787829746] req@ffff907d9114b840 x1874670011811328/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829746.421090:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011811328/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829746.421096:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829746.421099:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829746.421101:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011811328:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829746.421126:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011811456:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829746.421141:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829746.421148:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829746/real 1787829746] req@ffff907d9114a580 x1874670011811456/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829746.421156:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011811456/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829746.421163:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829746.421165:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829746.421167:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011811456:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829746.421172:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011811584:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829746.421183:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829746.421190:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829746/real 1787829746] req@ffff907d9114ad00 x1874670011811584/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829746.421198:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011811584/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829746.421203:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829746.421205:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829746.421207:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011811584:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829746.548751:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829746.548803:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829747.572744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829747.572847:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829748.596773:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829748.596875:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829749.620750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829749.620875:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829750.644677:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829750.644742:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829751.540815:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829751.540829:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829751.540835:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829751.540851:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5155 00000100:00080000:3.0:1787829751.540867:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829751.540878:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829751.540943:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011811712/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829751.540975:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829751.540979:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829751.540980:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829751.540984:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5155 00000100:00080000:3.0:1787829751.540988:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829751.540991:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829751.541002:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829751.541014:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011811840/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829751.541023:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829751.541027:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829751.541028:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829751.541031:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5155 00000100:00080000:3.0:1787829751.541035:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829751.541041:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829751.541043:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011811712:192.168.203.104@tcp:250:kworker.0 00000008:00100000:3.0:1787829751.541047:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829751.541058:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011811968/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829751.541066:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829751.541069:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829751.541070:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829751.541073:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5155 00000100:00080000:3.0:1787829751.541077:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829751.541080:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829751.541084:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829751.541094:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011812096/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829751.541130:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829751.541222:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829751/real 1787829751] req@ffff907da562f840 x1874670011811712/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829751.541259:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011811712/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829751.541275:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829751.541280:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829751.541286:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011811712:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829751.541322:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011811840:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829751.541365:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829751.541378:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829751/real 1787829751] req@ffff907da562e940 x1874670011811840/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829751.541389:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011811840/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829751.541397:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829751.541399:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829751.541402:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011811840:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829751.541430:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011811968:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829751.541446:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829751.541454:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829751/real 1787829751] req@ffff907da562c000 x1874670011811968/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829751.541464:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011811968/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829751.541471:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829751.541473:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829751.541476:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011811968:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829751.541480:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011812096:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829751.541492:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829751.541498:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829751/real 1787829751] req@ffff907da562f0c0 x1874670011812096/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829751.541517:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011812096/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829751.541534:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829751.541536:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829751.541539:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011812096:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829751.668721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829751.668792:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829752.692758:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829752.692845:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829753.716714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829753.716792:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829754.740717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829754.740743:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829755.764707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829755.764783:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829756.660687:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829756.660695:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829756.660698:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829756.660706:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5160 00000100:00080000:2.0:1787829756.660715:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829756.660721:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829756.660752:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011812224/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829756.660773:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829756.660776:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829756.660778:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829756.660780:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5160 00000100:00080000:2.0:1787829756.660783:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829756.660786:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829756.660794:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829756.660810:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011812224:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829756.660810:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011812352/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829756.660824:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829756.660826:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829756.660829:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829756.660832:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5160 00000100:00080000:2.0:1787829756.660834:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829756.660837:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829756.660841:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829756.660851:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011812480/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829756.660857:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829756.660860:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829756.660861:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829756.660863:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5160 00000100:00080000:2.0:1787829756.660866:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829756.660867:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829756.660871:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787829756.660876:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829756.660881:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011812608/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829756.660920:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829756/real 1787829756] req@ffff907d9114b480 x1874670011812224/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829756.660935:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011812224/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829756.660943:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829756.660946:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829756.660948:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011812224:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829756.660966:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011812352:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829756.660984:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829756.660994:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829756/real 1787829756] req@ffff907d91149680 x1874670011812352/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829756.661003:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011812352/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829756.661009:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829756.661011:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829756.661013:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011812352:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829756.661018:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011812480:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829756.661029:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829756.661035:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829756/real 1787829756] req@ffff907d9114bc00 x1874670011812480/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829756.661043:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011812480/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829756.661048:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829756.661050:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829756.661051:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011812480:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829756.661055:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011812608:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829756.661065:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829756.661071:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829756/real 1787829756] req@ffff907d911483c0 x1874670011812608/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829756.661078:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011812608/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829756.661083:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829756.661085:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829756.661087:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011812608:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829756.788709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829756.788771:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829757.812718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829757.812787:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829758.836746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829758.836818:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829759.860730:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829759.860800:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829760.884735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829760.884782:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829761.780769:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829761.780780:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829761.780785:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829761.780797:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5165 00000100:00080000:1.0:1787829761.780810:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829761.780817:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829761.780870:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011812736/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829761.780908:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829761.780911:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829761.780913:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829761.780916:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5165 00000100:00080000:1.0:1787829761.780919:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829761.780923:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829761.780934:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829761.780947:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011812864/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829761.780957:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829761.780962:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011812736:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829761.780962:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829761.780966:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829761.780974:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5165 00000100:00080000:1.0:1787829761.780976:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829761.780978:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829761.780982:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829761.780991:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011812992/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829761.780998:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829761.781001:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829761.781003:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829761.781005:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5165 00000100:00080000:1.0:1787829761.781007:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829761.781009:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829761.781013:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829761.781025:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011813120/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829761.781042:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829761.781086:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829761/real 1787829761] req@ffff907d908a3840 x1874670011812736/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829761.781102:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011812736/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829761.781111:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829761.781114:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829761.781117:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011812736:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829761.781139:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011812864:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829761.781155:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829761.781163:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829761/real 1787829761] req@ffff907d908a21c0 x1874670011812864/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829761.781171:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011812864/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829761.781177:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829761.781180:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829761.781182:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011812864:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829761.781185:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011812992:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829761.781198:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829761.781204:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829761/real 1787829761] req@ffff907d908a2d00 x1874670011812992/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829761.781212:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011812992/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829761.781235:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829761.781238:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829761.781240:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011812992:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829761.781244:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011813120:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829761.781259:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829761.781266:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829761/real 1787829761] req@ffff907d908a2940 x1874670011813120/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829761.781274:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011813120/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829761.781280:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829761.781282:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829761.781284:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011813120:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829761.908742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829761.908816:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829762.932743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829762.932822:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829763.956685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829763.956749:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829764.980706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829764.980722:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829766.004668:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829766.004733:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829766.900724:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829766.900732:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829766.900736:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829766.900744:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5170 00000100:00080000:0.0:1787829766.900754:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829766.900760:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829766.900793:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011813248/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829766.900819:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829766.900823:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829766.900824:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829766.900827:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5170 00000100:00080000:0.0:1787829766.900829:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829766.900832:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829766.900839:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829766.900848:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011813376/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829766.900855:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829766.900858:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829766.900860:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829766.900862:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5170 00000100:00080000:0.0:1787829766.900865:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829766.900867:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829766.900870:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829766.900877:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011813504/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829766.900884:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011813248:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829766.900887:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829766.900893:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829766.900894:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829766.900897:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5170 00000100:00080000:0.0:1787829766.900899:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829766.900901:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829766.900905:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829766.900913:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011813632/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829766.900944:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829766.900978:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829766/real 1787829766] req@ffff907d90845e00 x1874670011813248/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829766.900992:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011813248/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829766.901000:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829766.901003:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829766.901006:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011813248:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829766.901011:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011813376:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829766.901026:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829766.901034:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829766/real 1787829766] req@ffff907d90844000 x1874670011813376/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829766.901042:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011813376/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829766.901048:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829766.901049:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829766.901051:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011813376:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829766.901069:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011813504:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829766.901081:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829766.901088:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829766/real 1787829766] req@ffff907d908443c0 x1874670011813504/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829766.901095:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011813504/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829766.901105:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829766.901107:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829766.901109:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011813504:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829766.901112:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011813632:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829766.901121:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829766.901127:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829766/real 1787829766] req@ffff907d90844b40 x1874670011813632/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829766.901134:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011813632/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829766.901139:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829766.901141:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829766.901143:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011813632:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829767.028715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829767.028745:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829768.052728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829768.052882:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829769.076706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829769.076739:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829770.100721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829770.100793:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829771.124742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829771.124809:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829772.020770:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829772.020783:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829772.020788:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829772.020800:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5175 00000100:00080000:0.0:1787829772.020814:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829772.020823:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829772.020875:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011813760/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829772.020908:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829772.020912:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829772.020914:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829772.020918:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5175 00000100:00080000:0.0:1787829772.020922:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829772.020926:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829772.020939:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829772.020959:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011813888/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829772.020963:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011813760:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829772.020980:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829772.020984:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829772.020987:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829772.020992:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5175 00000100:00080000:0.0:1787829772.020997:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829772.021002:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829772.021008:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829772.021033:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011814016/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829772.021044:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787829772.021046:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829772.021052:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829772.021054:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829772.021060:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5175 00000100:00080000:0.0:1787829772.021064:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829772.021068:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829772.021074:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829772.021087:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011814144/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829772.021109:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829772/real 1787829772] req@ffff907d908470c0 x1874670011813760/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829772.021130:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011813760/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829772.021178:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829772.021184:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829772.021189:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011813760:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829772.021219:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011813888:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829772.021250:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829772.021265:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829772/real 1787829772] req@ffff907d90846580 x1874670011813888/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829772.021279:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011813888/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829772.021294:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829772.021299:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829772.021303:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011813888:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829772.021310:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011814016:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829772.021337:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829772.021356:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829772/real 1787829772] req@ffff907d90847c00 x1874670011814016/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829772.021367:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011814016/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829772.021375:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829772.021377:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829772.021380:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011814016:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829772.021385:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011814144:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829772.021407:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829772.021428:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829772/real 1787829772] req@ffff907d90844b40 x1874670011814144/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829772.021443:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011814144/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829772.021453:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829772.021456:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829772.021460:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011814144:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829772.148727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829772.148755:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829773.172654:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829773.172824:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829774.196747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829774.196807:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829775.220712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829775.220749:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829776.244770:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829776.244841:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829777.140696:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829777.140703:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829777.140706:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829777.140713:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5180 00000100:00080000:2.0:1787829777.140727:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829777.140732:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829777.140760:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011814272/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829777.140783:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829777.140786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829777.140787:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829777.140790:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5180 00000100:00080000:2.0:1787829777.140792:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829777.140795:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829777.140802:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829777.140810:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011814400/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829777.140817:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829777.140819:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829777.140820:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829777.140823:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5180 00000100:00080000:2.0:1787829777.140825:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829777.140827:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829777.140830:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829777.140839:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011814272:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829777.140847:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011814528/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829777.140857:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829777.140859:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829777.140860:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829777.140862:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5180 00000100:00080000:2.0:1787829777.140864:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829777.140866:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829777.140868:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829777.140876:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011814656/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829777.140969:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011814400:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829777.141005:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011814528:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829777.141020:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011814656:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829777.268719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829777.268799:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829778.292737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829778.292804:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829779.316736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829779.316771:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787829780.212836:0:2338:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:3.0:1787829780.212881:0:2338:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787829780.212900:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787829780.212944:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787829780.212956:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:3.0:1787829780.212966:0:2338:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:0.0:1787829780.212994:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829777/real 1787829780] req@ffff907d91149680 x1874670011814528/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829780.213026:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011814528/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829780.213037:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829780.213043:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829780.213047:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011814528:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829780.213055:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829777/real 1787829780] req@ffff907d9114b480 x1874670011814656/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829780.213067:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011814656/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829780.213074:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829780.213077:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829780.213080:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011814656:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829780.213087:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829777/real 1787829780] req@ffff907d911483c0 x1874670011814272/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829780.213115:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011814272/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829780.213123:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829780.213126:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829780.213130:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011814272:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829780.213137:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829777/real 1787829780] req@ffff907d9114bc00 x1874670011814400/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829780.213148:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011814400/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829780.213155:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829780.213158:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829780.213161:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011814400:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829780.340742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829780.340801:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829781.364686:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829781.364742:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829782.260732:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829782.260744:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829782.260749:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829782.260761:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5186 00000100:00080000:3.0:1787829782.260773:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829782.260782:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829782.260822:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011814784/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829782.260858:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829782.260862:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829782.260864:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829782.260868:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5186 00000100:00080000:3.0:1787829782.260873:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829782.260877:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829782.260889:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829782.260902:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011814912/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829782.260913:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829782.260916:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829782.260917:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829782.260921:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5186 00000100:00080000:3.0:1787829782.260925:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829782.260929:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829782.260933:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829782.260945:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011815040/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829782.260954:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829782.260958:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829782.260959:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829782.260962:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5186 00000100:00080000:3.0:1787829782.260966:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829782.260970:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829782.260974:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829782.260984:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011815168/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829782.261095:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011814784:192.168.203.104@tcp:250:kworker.0 00000800:00000100:0.0:1787829782.261173:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829782.261215:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829782/real 1787829782] req@ffff907da562c3c0 x1874670011814784/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829782.261231:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011814784/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829782.261240:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829782.261244:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829782.261248:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011814784:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829782.261255:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011814912:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829782.261279:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829782.261291:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829782/real 1787829782] req@ffff907da562fc00 x1874670011814912/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829782.261303:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011814912/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829782.261317:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829782.261320:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829782.261323:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011814912:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829782.261329:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011815040:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829782.261350:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829782.261362:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829782/real 1787829782] req@ffff907da562d2c0 x1874670011815040/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829782.261373:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011815040/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829782.261381:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829782.261384:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829782.261387:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011815040:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829782.261393:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011815168:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829782.261410:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829782.261419:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829782/real 1787829782] req@ffff907da562e580 x1874670011815168/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829782.261430:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011815168/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829782.261438:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829782.261441:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829782.261445:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011815168:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829782.388727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829782.388773:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829783.412687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829783.412751:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829784.436726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829784.436793:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829785.460687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829785.460781:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829786.484759:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829786.484820:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829787.380735:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829787.380747:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829787.380751:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829787.380769:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5191 00000100:00080000:3.0:1787829787.380782:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829787.380790:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829787.380833:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011815296/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829787.380865:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829787.380868:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829787.380870:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829787.380873:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5191 00000100:00080000:3.0:1787829787.380876:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829787.380878:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829787.380888:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829787.380898:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011815424/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829787.380905:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829787.380910:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829787.380911:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829787.380915:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5191 00000100:00080000:3.0:1787829787.380917:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829787.380920:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829787.380923:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829787.380930:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011815552/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829787.380938:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829787.380939:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011815296:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829787.380941:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829787.380944:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829787.380950:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5191 00000100:00080000:3.0:1787829787.380952:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829787.380955:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829787.380958:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829787.380967:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011815680/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829787.381018:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829787.381091:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829787/real 1787829787] req@ffff907da562cf00 x1874670011815296/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829787.381112:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011815296/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829787.381122:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829787.381126:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829787.381130:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011815296:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829787.381139:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011815424:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829787.381165:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829787.381179:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829787/real 1787829787] req@ffff907da562c780 x1874670011815424/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829787.381189:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011815424/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829787.381195:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829787.381197:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829787.381200:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011815424:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829787.381224:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011815552:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829787.381242:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829787.381254:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829787/real 1787829787] req@ffff907da562ed00 x1874670011815552/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829787.381266:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011815552/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829787.381275:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829787.381279:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829787.381283:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011815552:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829787.381289:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011815680:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829787.381306:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829787.381318:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829787/real 1787829787] req@ffff907da562f480 x1874670011815680/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829787.381330:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011815680/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829787.381339:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829787.381343:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829787.381346:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011815680:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829787.508752:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829787.508882:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829788.532729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829788.532796:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829789.556740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829789.556811:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829790.580752:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829790.580832:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829791.604756:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829791.604822:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829792.500756:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829792.500765:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829792.500769:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829792.500777:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5196 00000100:00080000:2.0:1787829792.500786:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829792.500808:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829792.500848:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011815808/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829792.500868:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829792.500870:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829792.500872:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829792.500875:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5196 00000100:00080000:2.0:1787829792.500877:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829792.500880:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829792.500888:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829792.500898:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011815936/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829792.500906:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829792.500910:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787829792.500912:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011815808:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829792.500912:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829792.500920:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5196 00000100:00080000:2.0:1787829792.500922:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829792.500924:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829792.500938:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829792.500947:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011816064/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829792.500954:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829792.500957:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829792.500958:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829792.500961:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5196 00000100:00080000:2.0:1787829792.500963:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829792.500965:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829792.500968:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829792.500975:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011816192/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829792.500986:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829792.501038:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829792/real 1787829792] req@ffff907d9114ad00 x1874670011815808/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829792.501058:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011815808/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829792.501079:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829792.501084:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829792.501089:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011815808:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829792.501108:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011815936:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829792.501127:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829792.501136:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829792/real 1787829792] req@ffff907d9114a580 x1874670011815936/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829792.501145:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011815936/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829792.501151:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829792.501153:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829792.501155:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011815936:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829792.501159:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011816064:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829792.501171:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829792.501177:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829792/real 1787829792] req@ffff907d9114b840 x1874670011816064/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829792.501185:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011816064/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829792.501190:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829792.501192:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829792.501194:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011816064:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829792.501198:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011816192:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829792.501209:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829792.501214:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829792/real 1787829792] req@ffff907d91149a40 x1874670011816192/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829792.501222:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011816192/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829792.501227:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829792.501228:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829792.501230:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011816192:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829792.628685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829792.628746:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829793.652661:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829793.652722:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829794.676755:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829794.676816:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829795.700704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829795.700760:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829796.724720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829796.724792:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829797.620681:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829797.620690:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829797.620694:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829797.620702:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5201 00000100:00080000:2.0:1787829797.620712:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829797.620718:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829797.620750:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011816320/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829797.620780:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829797.620783:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829797.620785:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829797.620788:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5201 00000100:00080000:2.0:1787829797.620790:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829797.620792:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829797.620800:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829797.620811:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011816448/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829797.620819:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829797.620822:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829797.620824:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829797.620832:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5201 00000100:00080000:2.0:1787829797.620834:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829797.620836:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829797.620841:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829797.620847:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011816320:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829797.620855:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011816576/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829797.620865:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829797.620867:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829797.620868:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829797.620871:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5201 00000100:00080000:2.0:1787829797.620873:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829797.620877:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829797.620880:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829797.620888:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011816704/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829797.620929:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829797.620999:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829797/real 1787829797] req@ffff907d911492c0 x1874670011816320/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829797.621034:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011816320/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829797.621044:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829797.621047:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829797.621051:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011816320:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829797.621057:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011816448:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829797.621081:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829797.621096:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829797/real 1787829797] req@ffff907d91148b40 x1874670011816448/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829797.621111:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011816448/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829797.621122:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829797.621126:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829797.621130:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011816448:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829797.621157:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011816576:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829797.621177:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829797.621187:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829797/real 1787829797] req@ffff907d9114a940 x1874670011816576/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829797.621196:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011816576/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829797.621202:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829797.621205:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829797.621207:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011816576:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829797.621212:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011816704:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829797.621232:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829797.621240:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829797/real 1787829797] req@ffff907d91148f00 x1874670011816704/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829797.621248:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011816704/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829797.621255:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829797.621258:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829797.621260:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011816704:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829797.748747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829797.748797:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829798.772699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829798.772791:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829799.796716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829799.796847:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829800.820728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829800.820777:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829801.844726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829801.844786:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829802.740748:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829802.740760:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829802.740765:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829802.740776:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5206 00000100:00080000:0.0:1787829802.740789:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829802.740796:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829802.740839:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011816832/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829802.740867:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829802.740870:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829802.740872:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829802.740875:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5206 00000100:00080000:0.0:1787829802.740877:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829802.740881:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829802.740891:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829802.740900:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011816960/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829802.740906:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011816832:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829802.740912:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829802.740917:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829802.740920:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829802.740925:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5206 00000100:00080000:0.0:1787829802.740928:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829802.740933:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829802.740951:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829802.740965:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011817088/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829802.740977:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829802.740981:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829802.740982:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829802.740987:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5206 00000100:00080000:0.0:1787829802.740991:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829802.740994:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829802.741000:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787829802.741001:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829802.741018:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011817216/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829802.741052:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829802/real 1787829802] req@ffff907d908443c0 x1874670011816832/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829802.741067:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011816832/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829802.741076:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829802.741080:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829802.741084:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011816832:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829802.741106:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011816960:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829802.741122:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829802.741131:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829802/real 1787829802] req@ffff907d90844000 x1874670011816960/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829802.741140:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011816960/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829802.741149:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829802.741152:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829802.741156:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011816960:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829802.741162:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011817088:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829802.741185:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829802.741197:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829802/real 1787829802] req@ffff907d90845e00 x1874670011817088/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829802.741209:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011817088/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829802.741216:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829802.741218:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829802.741220:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011817088:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829802.741224:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011817216:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829802.741243:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829802.741257:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829802/real 1787829802] req@ffff907d908452c0 x1874670011817216/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829802.741270:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011817216/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829802.741279:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829802.741283:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829802.741286:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011817216:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829802.868685:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829802.868730:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829803.892743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829803.892775:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829804.916671:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829804.916768:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829805.940738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829805.940813:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829806.964708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829806.964781:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829807.860767:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829807.860778:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829807.860783:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829807.860795:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5211 00000100:00080000:2.0:1787829807.860810:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829807.860818:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829807.860861:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011817344/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829807.860892:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829807.860895:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829807.860897:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829807.860912:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5211 00000100:00080000:2.0:1787829807.860918:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829807.860924:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829807.860938:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829807.860945:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011817344:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829807.860958:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011817472/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829807.860973:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829807.860976:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829807.860979:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829807.860986:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5211 00000100:00080000:2.0:1787829807.860996:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829807.861002:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829807.861008:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829807.861018:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011817600/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829807.861024:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787829807.861034:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829807.861038:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829807.861039:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829807.861044:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5211 00000100:00080000:2.0:1787829807.861048:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829807.861052:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829807.861058:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829807.861069:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011817728/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829807.861089:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829807/real 1787829807] req@ffff907d91148f00 x1874670011817344/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829807.861109:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011817344/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829807.861123:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829807.861127:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829807.861130:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011817344:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829807.861151:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011817472:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829807.861180:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829807.861189:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829807/real 1787829807] req@ffff907d9114a940 x1874670011817472/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829807.861198:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011817472/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829807.861203:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829807.861206:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829807.861208:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011817472:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829807.861213:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011817600:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829807.861225:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829807.861231:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829807/real 1787829807] req@ffff907d91148b40 x1874670011817600/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829807.861239:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011817600/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829807.861244:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829807.861246:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829807.861249:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011817600:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829807.861256:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011817728:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829807.861275:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829807.861287:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829807/real 1787829807] req@ffff907d911492c0 x1874670011817728/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829807.861301:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011817728/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829807.861310:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829807.861313:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829807.861315:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011817728:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829807.988712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829807.988803:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829809.012754:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829809.012853:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829810.036708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829810.036734:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829811.060728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829811.060755:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829812.084717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829812.084785:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829812.980776:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829812.980792:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829812.980798:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829812.980812:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5216 00000100:00080000:0.0:1787829812.980827:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829812.980835:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829812.980901:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011817856/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829812.980938:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829812.980942:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829812.980945:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829812.980951:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5216 00000100:00080000:0.0:1787829812.980956:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829812.980962:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829812.980976:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829812.980992:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011817856:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829812.980998:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011817984/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829812.981017:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829812.981021:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829812.981023:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829812.981029:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5216 00000100:00080000:0.0:1787829812.981033:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829812.981039:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829812.981044:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829812.981060:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011818112/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829812.981061:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:0.0:1787829812.981079:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829812.981084:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829812.981086:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829812.981092:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5216 00000100:00080000:0.0:1787829812.981097:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829812.981104:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829812.981109:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829812/real 1787829812] req@ffff907d90844780 x1874670011817856/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787829812.981112:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829812.981136:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011817856/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829812.981140:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011818240/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829812.981159:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829812.981168:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829812.981172:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011817856:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829812.981194:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011817984:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829812.981210:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829812.981219:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829812/real 1787829812] req@ffff907d90845680 x1874670011817984/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829812.981227:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011817984/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829812.981232:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829812.981235:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829812.981237:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011817984:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829812.981241:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011818112:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829812.981252:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829812.981258:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829812/real 1787829812] req@ffff907d90846d00 x1874670011818112/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829812.981266:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011818112/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829812.981271:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829812.981273:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829812.981275:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011818112:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829812.981279:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011818240:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829812.981288:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829812.981294:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829812/real 1787829812] req@ffff907d90845a40 x1874670011818240/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829812.981301:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011818240/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829812.981307:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829812.981308:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829812.981310:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011818240:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829813.108747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829813.108796:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829814.132711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829814.132773:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829815.156707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829815.156863:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829816.180700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829816.180911:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829817.204713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829817.204732:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829818.100719:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829818.100727:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829818.100731:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829818.100740:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5221 00000100:00080000:2.0:1787829818.100749:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829818.100755:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829818.100786:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011818368/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829818.100822:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829818.100827:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829818.100843:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829818.100849:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5221 00000100:00080000:2.0:1787829818.100853:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829818.100859:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829818.100864:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011818368:192.168.203.104@tcp:250:kworker.0 00000008:00100000:2.0:1787829818.100871:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829818.100886:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011818496/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829818.100895:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829818.100898:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829818.100899:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829818.100902:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5221 00000100:00080000:2.0:1787829818.100905:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829818.100908:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829818.100912:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829818.100923:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011818624/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829818.100924:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787829818.100933:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829818.100936:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829818.100938:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829818.100942:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5221 00000100:00080000:2.0:1787829818.100945:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829818.100948:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829818.100951:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829818.100959:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011818752/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829818.100967:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829818/real 1787829818] req@ffff907d91149a40 x1874670011818368/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829818.100983:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011818368/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829818.100991:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829818.100994:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829818.100997:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011818368:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829818.101014:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011818496:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829818.101029:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829818.101037:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829818/real 1787829818] req@ffff907d9114b840 x1874670011818496/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829818.101053:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011818496/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829818.101059:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829818.101062:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829818.101064:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011818496:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829818.101068:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011818624:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829818.101080:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829818.101090:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829818/real 1787829818] req@ffff907d9114a580 x1874670011818624/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829818.101112:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011818624/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829818.101121:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829818.101125:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829818.101128:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011818624:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829818.101135:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011818752:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829818.101153:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829818.101161:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829818/real 1787829818] req@ffff907d9114ad00 x1874670011818752/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829818.101169:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011818752/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829818.101174:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829818.101177:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829818.101179:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011818752:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829818.228671:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829818.228730:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829819.252746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829819.252810:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829820.276713:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829820.276738:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829821.300729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829821.300786:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829822.324729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829822.324775:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829823.220750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829823.220762:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829823.220767:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829823.220777:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5226 00000100:00080000:1.0:1787829823.220790:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829823.220806:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829823.220845:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011818880/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829823.220867:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829823.220870:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829823.220871:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829823.220874:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5226 00000100:00080000:1.0:1787829823.220876:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829823.220879:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829823.220888:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829823.220902:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011819008/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829823.220914:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829823.220916:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829823.220917:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829823.220922:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5226 00000100:00100000:0.0:1787829823.220923:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011818880:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829823.220925:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829823.220930:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829823.220935:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829823.220944:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011819136/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829823.220950:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829823.220953:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829823.220954:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829823.220956:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5226 00000100:00080000:1.0:1787829823.220957:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829823.220959:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829823.220962:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829823.220969:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011819264/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829823.221005:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829823.221063:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829823/real 1787829823] req@ffff907d908a1e00 x1874670011818880/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829823.221083:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011818880/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829823.221099:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829823.221110:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829823.221114:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011818880:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829823.221142:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011819008:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829823.221165:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829823.221177:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829823/real 1787829823] req@ffff907d908a0000 x1874670011819008/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829823.221197:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011819008/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829823.221205:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829823.221209:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829823.221212:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011819008:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829823.221218:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011819136:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829823.221247:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829823.221259:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829823/real 1787829823] req@ffff907d908a2940 x1874670011819136/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829823.221271:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011819136/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829823.221279:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829823.221282:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829823.221285:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011819136:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829823.221291:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011819264:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829823.221311:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829823.221326:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829823/real 1787829823] req@ffff907d908a2d00 x1874670011819264/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829823.221333:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011819264/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829823.221338:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829823.221340:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829823.221342:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011819264:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829823.348732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829823.348812:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829824.372728:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829824.372774:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829825.396719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829825.396773:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829826.420740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829826.420818:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829827.444721:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829827.444753:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829828.340792:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829828.340801:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829828.340805:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829828.340821:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5232 00000100:00080000:2.0:1787829828.340831:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829828.340838:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829828.340870:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011819392/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829828.340901:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829828.340904:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829828.340906:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829828.340910:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5232 00000100:00080000:2.0:1787829828.340913:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829828.340917:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829828.340926:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829828.340940:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011819520/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829828.340943:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011819392:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829828.340958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829828.340962:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829828.340963:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829828.340968:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5232 00000100:00080000:2.0:1787829828.340971:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829828.340976:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829828.340981:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829828.340991:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011819648/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829828.340999:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787829828.341012:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829828.341018:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829828.341021:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829828.341026:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5232 00000100:00080000:2.0:1787829828.341030:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829828.341033:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829828.341038:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829828.341045:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829828/real 1787829828] req@ffff907d9114bc00 x1874670011819392/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829828.341052:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011819776/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829828.341062:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011819392/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829828.341073:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829828.341078:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829828.341082:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011819392:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829828.341104:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011819520:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829828.341127:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829828.341140:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829828/real 1787829828] req@ffff907d911483c0 x1874670011819520/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829828.341153:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011819520/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829828.341160:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829828.341163:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829828.341166:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011819520:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829828.341171:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011819648:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829828.341189:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829828.341200:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829828/real 1787829828] req@ffff907d9114b480 x1874670011819648/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829828.341210:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011819648/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829828.341217:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829828.341220:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829828.341223:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011819648:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829828.341229:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011819776:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829828.341247:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829828.341257:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829828/real 1787829828] req@ffff907d91149680 x1874670011819776/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829828.341276:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011819776/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829828.341291:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829828.341294:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829828.341297:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011819776:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829828.468706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829828.468782:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829829.492789:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829829.492809:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787829830.516707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829830.516778:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829831.540761:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829831.540809:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829832.564748:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829832.564824:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829833.460719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829833.460737:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829833.460742:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829833.460750:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5237 00000100:00080000:2.0:1787829833.460760:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829833.460765:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829833.460795:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011819904/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829833.460822:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829833.460826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829833.460828:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829833.460833:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5237 00000100:00080000:2.0:1787829833.460837:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829833.460842:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829833.460852:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829833.460876:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011819904:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829833.460876:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011820032/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829833.460911:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829833.460913:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829833.460914:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829833.460919:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5237 00000100:00080000:2.0:1787829833.460922:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829833.460925:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829833.460929:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829833.460938:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011820160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829833.460949:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829833.460952:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829833.460954:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829833.460957:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5237 00000100:00080000:2.0:1787829833.460962:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829833.460967:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:1.0:1787829833.460972:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:2.0:1787829833.460972:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829833.460988:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011820288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829833.461016:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829833/real 1787829833] req@ffff907d91149680 x1874670011819904/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829833.461031:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011819904/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829833.461039:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829833.461042:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829833.461045:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011819904:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829833.461065:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011820032:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829833.461084:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829833.461096:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829833/real 1787829833] req@ffff907d9114b480 x1874670011820032/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829833.461107:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011820032/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829833.461116:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829833.461119:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829833.461122:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011820032:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829833.461128:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011820160:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829833.461145:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829833.461155:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829833/real 1787829833] req@ffff907d911483c0 x1874670011820160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829833.461173:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011820160/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829833.461181:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829833.461184:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829833.461188:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011820160:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829833.461194:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011820288:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829833.461209:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829833.461216:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829833/real 1787829833] req@ffff907d9114bc00 x1874670011820288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829833.461222:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011820288/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829833.461228:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829833.461230:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829833.461232:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011820288:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829833.588712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829833.588781:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829834.612743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829834.612793:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829835.636741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829835.636782:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829836.660748:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829836.660843:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829837.684774:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829837.684786:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829838.580678:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829838.580702:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829838.580708:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829838.580720:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5242 00000100:00080000:2.0:1787829838.580735:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829838.580743:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829838.580791:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011820416/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829838.580825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829838.580828:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829838.580830:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829838.580833:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5242 00000100:00080000:2.0:1787829838.580838:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829838.580842:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829838.580855:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829838.580871:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011820544/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829838.580873:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011820416:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829838.580895:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829838.580900:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829838.580902:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829838.580908:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5242 00000100:00080000:2.0:1787829838.580914:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829838.580919:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829838.580925:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829838.580937:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011820672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829838.580949:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787829838.580960:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829838.580964:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829838.580967:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829838.580972:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5242 00000100:00080000:2.0:1787829838.580978:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829838.580982:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829838.580987:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829838.580990:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829838/real 1787829838] req@ffff907d9114ad00 x1874670011820416/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829838.581009:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011820800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829838.581010:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011820416/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829838.581079:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829838.581085:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829838.581090:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011820416:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829838.581110:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011820544:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829838.581125:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829838.581133:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829838/real 1787829838] req@ffff907d9114a580 x1874670011820544/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829838.581145:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011820544/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829838.581152:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829838.581156:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829838.581159:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011820544:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829838.581165:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011820672:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829838.581184:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829838.581192:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829838/real 1787829838] req@ffff907d9114b840 x1874670011820672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829838.581206:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011820672/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829838.581211:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829838.581213:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829838.581215:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011820672:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829838.581220:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011820800:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829838.581231:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829838.581237:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829838/real 1787829838] req@ffff907d91149a40 x1874670011820800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829838.581243:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011820800/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829838.581247:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829838.581249:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829838.581251:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011820800:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829838.708703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829838.708784:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829839.732703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829839.732780:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829840.756734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829840.756810:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829841.780693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829841.780783:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829842.804734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829842.804793:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829843.700701:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829843.700716:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829843.700722:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829843.700735:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5247 00000100:00080000:1.0:1787829843.700750:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829843.700759:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829843.700806:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011820928/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829843.700839:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829843.700843:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829843.700846:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829843.700851:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5247 00000100:00080000:1.0:1787829843.700856:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829843.700861:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829843.700868:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011820928:192.168.203.104@tcp:250:kworker.0 00000008:00100000:1.0:1787829843.700873:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829843.700892:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011821056/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829843.700905:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829843.700908:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829843.700911:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829843.700916:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5247 00000100:00080000:1.0:1787829843.700920:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829843.700925:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829843.700932:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829843.700945:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011821184/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829843.700956:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829843.700960:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829843.700963:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829843.700967:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5247 00000100:00080000:1.0:1787829843.700972:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829843.700976:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829843.700982:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829843.700990:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011821056:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829843.700996:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011821312/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829843.701023:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011821184:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829843.701047:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011821312:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829843.828700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829843.828743:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829844.852825:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829844.852849:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787829845.876751:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829845.876774:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787829846.772910:0:2335:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:0.0:1787829846.772968:0:2335:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787829846.772986:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787829846.773035:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787829846.773047:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787829846.773056:0:2335:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:2.0:1787829846.773093:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829843/real 1787829846] req@ffff907d908a2580 x1874670011821312/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829846.773124:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011821312/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829846.773150:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829846.773156:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829846.773161:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011821312:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829846.773170:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829843/real 1787829846] req@ffff907d908a3840 x1874670011821056/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829846.773189:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011821056/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829846.773197:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829846.773200:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829846.773203:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011821056:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829846.773209:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829843/real 1787829846] req@ffff907d908a1680 x1874670011821184/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829846.773219:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011821184/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829846.773226:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829846.773229:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829846.773232:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011821184:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829846.773238:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829843/real 1787829846] req@ffff907d908a21c0 x1874670011820928/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829846.773250:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011820928/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829846.773256:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829846.773259:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829846.773262:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011820928:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829846.900705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829846.900756:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829847.924738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829847.924830:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829848.820712:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829848.820724:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829848.820729:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829848.820739:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5252 00000100:00080000:1.0:1787829848.820751:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829848.820759:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829848.820798:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a12c0 x1874670011821440/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829848.820829:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829848.820832:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829848.820833:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829848.820836:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5252 00000100:00080000:1.0:1787829848.820839:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829848.820841:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829848.820852:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829848.820861:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0f00 x1874670011821568/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829848.820870:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829848.820873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829848.820876:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829848.820879:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011821440:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829848.820880:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5252 00000100:00080000:1.0:1787829848.820890:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829848.820896:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829848.820902:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829848.820918:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011821696/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829848.820929:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829848.820934:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829848.820936:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829848.820940:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5252 00000100:00080000:1.0:1787829848.820945:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829848.820950:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829848.820955:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829848.820978:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011821824/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829848.820979:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829848.821037:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829848/real 1787829848] req@ffff907d908a12c0 x1874670011821440/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829848.821060:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a12c0 x1874670011821440/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829848.821073:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829848.821077:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829848.821082:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a12c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011821440:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829848.821114:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011821568:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829848.821141:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829848.821157:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829848/real 1787829848] req@ffff907d908a0f00 x1874670011821568/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829848.821171:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0f00 x1874670011821568/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829848.821180:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829848.821184:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829848.821188:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011821568:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829848.821195:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011821696:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829848.821218:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829848.821230:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829848/real 1787829848] req@ffff907d908a21c0 x1874670011821696/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829848.821244:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011821696/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829848.821254:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829848.821257:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829848.821261:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011821696:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829848.821268:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011821824:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829848.821289:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829848.821301:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829848/real 1787829848] req@ffff907d908a1680 x1874670011821824/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829848.821312:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011821824/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829848.821321:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829848.821324:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829848.821327:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011821824:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829848.948745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829848.948823:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829849.972729:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829849.972761:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829850.996727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829850.996780:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829852.020766:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829852.020806:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:0.0:1787829853.044731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829853.044819:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829853.940737:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829853.940748:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829853.940754:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829853.940766:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5257 00000100:00080000:0.0:1787829853.940778:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829853.940786:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829853.940828:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844f00 x1874670011821952/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829853.940854:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829853.940857:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829853.940858:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829853.940862:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5257 00000100:00080000:0.0:1787829853.940865:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829853.940867:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829853.940878:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829853.940887:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846940 x1874670011822080/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829853.940897:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829853.940900:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829853.940902:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829853.940907:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5257 00000100:00080000:0.0:1787829853.940910:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829853.940917:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829853.940917:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011821952:192.168.203.104@tcp:250:kworker.0 00000008:00100000:0.0:1787829853.940933:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829853.940942:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847480 x1874670011822208/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829853.940951:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829853.940954:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829853.940955:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829853.940959:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5257 00000100:00080000:0.0:1787829853.940962:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829853.940965:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829853.940968:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829853.940978:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011822336/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829853.940992:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829853.941032:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829853/real 1787829853] req@ffff907d90844f00 x1874670011821952/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829853.941051:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844f00 x1874670011821952/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829853.941069:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829853.941073:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829853.941076:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011821952:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829853.941101:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011822080:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829853.941118:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829853.941127:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829853/real 1787829853] req@ffff907d90846940 x1874670011822080/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829853.941135:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846940 x1874670011822080/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829853.941141:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829853.941143:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829853.941145:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011822080:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829853.941150:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011822208:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829853.941162:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829853.941168:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829853/real 1787829853] req@ffff907d90847480 x1874670011822208/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829853.941176:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847480 x1874670011822208/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829853.941185:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829853.941187:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829853.941189:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011822208:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829853.941193:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011822336:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829853.941203:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829853.941208:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829853/real 1787829853] req@ffff907d90845a40 x1874670011822336/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829853.941215:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011822336/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829853.941220:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829853.941222:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829853.941224:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011822336:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829854.068774:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829854.068842:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829855.092840:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829855.092897:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787829856.116737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829856.116791:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829857.140670:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829857.141685:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829858.164777:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829858.164841:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829859.060776:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829859.060785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829859.060789:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829859.060798:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5262 00000100:00080000:2.0:1787829859.060808:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829859.060815:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829859.060854:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011822464/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829859.060890:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829859.060894:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829859.060896:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829859.060900:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5262 00000100:00080000:2.0:1787829859.060904:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829859.060908:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829859.060917:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829859.060933:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011822592/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829859.060943:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829859.060947:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829859.060949:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829859.060953:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5262 00000100:00080000:2.0:1787829859.060957:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829859.060962:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011822464:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829859.060965:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829859.060974:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829859.060987:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011822720/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829859.060998:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829859.061002:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829859.061004:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829859.061008:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5262 00000100:00080000:2.0:1787829859.061012:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829859.061016:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829859.061021:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829859.061030:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011822848/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829859.061040:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829859.061086:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829859/real 1787829859] req@ffff907d911492c0 x1874670011822464/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829859.061107:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011822464/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829859.061119:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829859.061123:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829859.061128:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011822464:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829859.061136:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011822592:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829859.061164:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829859.061176:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829859/real 1787829859] req@ffff907d91148b40 x1874670011822592/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829859.061185:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011822592/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829859.061191:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829859.061193:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829859.061196:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011822592:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829859.061213:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011822720:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829859.061228:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829859.061235:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829859/real 1787829859] req@ffff907d9114a940 x1874670011822720/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829859.061243:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011822720/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829859.061248:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829859.061250:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829859.061252:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011822720:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829859.061256:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011822848:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829859.061266:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829859.061272:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829859/real 1787829859] req@ffff907d91148f00 x1874670011822848/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829859.061279:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011822848/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829859.061284:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829859.061286:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829859.061288:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011822848:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829859.188693:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829859.188763:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829860.212745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829860.212844:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829861.236708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829861.236769:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829862.260733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829862.260829:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829863.284741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829863.284803:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829864.180747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829864.180762:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829864.180768:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829864.180783:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5267 00000100:00080000:2.0:1787829864.180808:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829864.180819:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829864.180870:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011822976/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829864.180897:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829864.180900:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829864.180902:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829864.180905:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5267 00000100:00080000:2.0:1787829864.180908:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829864.180911:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829864.180922:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829864.180936:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011823104/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829864.180944:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829864.180947:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829864.180948:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829864.180951:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5267 00000100:00080000:2.0:1787829864.180952:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829864.180958:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829864.180959:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011822976:192.168.203.104@tcp:250:kworker.0 00000008:00100000:2.0:1787829864.180962:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829864.180972:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011823232/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829864.180979:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829864.180989:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829864.180990:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829864.180993:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5267 00000100:00080000:2.0:1787829864.180996:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829864.180999:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829864.181003:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829864.181012:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011823360/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829864.181037:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829864.181111:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829864/real 1787829864] req@ffff907d91148f00 x1874670011822976/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829864.181136:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011822976/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829864.181146:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829864.181149:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829864.181153:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011822976:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829864.181176:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011823104:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829864.181201:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829864.181217:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829864/real 1787829864] req@ffff907d9114a940 x1874670011823104/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829864.181231:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011823104/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829864.181242:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829864.181245:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829864.181249:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011823104:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829864.181254:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011823232:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829864.181272:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829864.181281:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829864/real 1787829864] req@ffff907d91148b40 x1874670011823232/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829864.181289:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011823232/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829864.181295:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829864.181297:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829864.181299:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011823232:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829864.181305:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011823360:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829864.181334:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829864.181347:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829864/real 1787829864] req@ffff907d911492c0 x1874670011823360/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829864.181361:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011823360/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829864.181373:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829864.181376:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829864.181379:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011823360:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829864.308732:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829864.308765:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829865.332736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829865.332803:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829866.356679:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829866.356742:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829867.380711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829867.380752:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829868.404687:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829868.404751:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829869.300705:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829869.300714:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829869.300717:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829869.300726:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5273 00000100:00080000:2.0:1787829869.300736:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829869.300741:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829869.300773:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011823488/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829869.300795:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829869.300799:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829869.300800:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829869.300803:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5273 00000100:00080000:2.0:1787829869.300806:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829869.300808:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829869.300817:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829869.300830:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011823616/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829869.300842:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829869.300848:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011823488:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829869.300849:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829869.300852:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829869.300862:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5273 00000100:00080000:2.0:1787829869.300865:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829869.300868:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829869.300873:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829869.300883:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011823744/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829869.300891:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829869.300893:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829869.300895:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829869.300897:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5273 00000100:00080000:2.0:1787829869.300903:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829869.300905:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829869.300908:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829869.300917:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011823872/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829869.300922:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829869.300966:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829869/real 1787829869] req@ffff907d91149a40 x1874670011823488/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829869.300985:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011823488/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829869.300993:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829869.300996:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829869.300999:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011823488:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829869.301017:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011823616:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829869.301034:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829869.301042:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829869/real 1787829869] req@ffff907d9114b840 x1874670011823616/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829869.301057:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011823616/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829869.301063:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829869.301066:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829869.301068:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011823616:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829869.301072:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011823744:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829869.301087:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829869.301094:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829869/real 1787829869] req@ffff907d9114a580 x1874670011823744/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829869.301101:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011823744/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829869.301107:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829869.301108:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829869.301110:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011823744:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829869.301114:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011823872:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829869.301124:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829869.301130:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829869/real 1787829869] req@ffff907d9114ad00 x1874670011823872/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829869.301137:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011823872/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829869.301142:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829869.301144:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829869.301146:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011823872:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829869.428706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829869.428773:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829870.452744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829870.452802:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829871.476745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829871.476773:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829872.500734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829872.500802:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829873.524733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829873.524762:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829874.420758:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829874.420769:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829874.420773:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829874.420785:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5278 00000100:00080000:0.0:1787829874.420797:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829874.420804:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829874.420852:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011824000/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829874.420877:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829874.420880:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829874.420881:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829874.420886:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5278 00000100:00080000:0.0:1787829874.420889:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829874.420894:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829874.420907:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829874.420923:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011824128/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829874.420929:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011824000:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829874.420944:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829874.420946:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829874.420948:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829874.420953:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5278 00000100:00080000:0.0:1787829874.420956:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829874.420966:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829874.420971:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829874.420982:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011824256/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829874.420989:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829874.420992:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829874.420994:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829874.420996:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5278 00000100:00080000:0.0:1787829874.420998:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829874.421000:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829874.421003:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829874.421013:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011824384/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829874.421017:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829874.421065:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829874/real 1787829874] req@ffff907d90846d00 x1874670011824000/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829874.421086:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011824000/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829874.421095:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829874.421099:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829874.421102:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011824000:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829874.421123:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011824128:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829874.421141:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829874.421149:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829874/real 1787829874] req@ffff907d90845680 x1874670011824128/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829874.421157:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011824128/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829874.421163:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829874.421166:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829874.421168:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011824128:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829874.421171:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011824256:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829874.421183:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829874.421189:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829874/real 1787829874] req@ffff907d90844780 x1874670011824256/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829874.421197:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011824256/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829874.421207:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829874.421209:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829874.421211:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011824256:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829874.421214:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011824384:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829874.421224:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829874.421229:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829874/real 1787829874] req@ffff907d908452c0 x1874670011824384/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829874.421237:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011824384/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829874.421242:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829874.421244:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829874.421246:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011824384:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829874.548703:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829874.548729:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829875.572739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829875.572825:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829876.596708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829876.596763:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829877.620722:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829877.620853:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829878.644702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829878.644775:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829879.540734:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829879.540744:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829879.540749:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829879.540760:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5283 00000100:00080000:0.0:1787829879.540771:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829879.540780:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829879.540821:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011824512/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829879.540860:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829879.540863:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829879.540864:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829879.540867:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5283 00000100:00080000:0.0:1787829879.540869:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829879.540871:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829879.540883:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829879.540893:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011824640/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829879.540901:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829879.540904:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829879.540906:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829879.540909:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5283 00000100:00080000:0.0:1787829879.540911:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829879.540917:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829879.540917:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011824512:192.168.203.104@tcp:250:kworker.0 00000008:00100000:0.0:1787829879.540925:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829879.540934:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011824768/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829879.540944:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829879.540949:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829879.540951:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829879.540955:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5283 00000100:00080000:0.0:1787829879.540958:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829879.540962:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829879.540967:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829879.540979:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011824896/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829879.540998:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829879.541041:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829879/real 1787829879] req@ffff907d90845e00 x1874670011824512/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829879.541058:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011824512/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829879.541069:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829879.541076:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829879.541083:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011824512:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829879.541093:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011824640:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829879.541129:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829879.541146:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829879/real 1787829879] req@ffff907d90844000 x1874670011824640/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829879.541157:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011824640/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829879.541163:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829879.541167:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829879.541169:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011824640:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829879.541194:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011824768:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829879.541210:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829879.541224:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829879/real 1787829879] req@ffff907d908443c0 x1874670011824768/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829879.541232:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011824768/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829879.541238:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829879.541240:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829879.541243:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011824768:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829879.541249:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011824896:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829879.541260:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829879.541274:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829879/real 1787829879] req@ffff907d90844b40 x1874670011824896/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829879.541282:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011824896/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829879.541288:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829879.541290:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829879.541299:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011824896:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829879.668715:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829879.668811:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829880.692692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829880.692765:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829881.716714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829881.716771:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829882.740683:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829882.740771:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829883.764706:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829883.764754:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829884.660751:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829884.660760:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829884.660763:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829884.660771:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5288 00000100:00080000:2.0:1787829884.660780:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829884.660786:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829884.660824:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011825024/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829884.660847:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829884.660850:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829884.660851:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829884.660854:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5288 00000100:00080000:2.0:1787829884.660857:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829884.660859:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829884.660867:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829884.660877:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011825152/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829884.660888:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829884.660892:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829884.660894:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829884.660898:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5288 00000100:00080000:2.0:1787829884.660902:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829884.660908:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829884.660913:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011825024:192.168.203.104@tcp:250:kworker.0 00000008:00100000:2.0:1787829884.660916:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829884.660933:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011825280/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829884.660941:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829884.660944:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829884.660945:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829884.660949:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5288 00000100:00080000:2.0:1787829884.660952:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829884.660954:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829884.660958:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829884.660968:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011825408/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829884.660979:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829884.661019:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829884/real 1787829884] req@ffff907d9114bc00 x1874670011825024/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829884.661033:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011825024/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829884.661042:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829884.661045:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829884.661048:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011825024:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829884.661053:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011825152:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829884.661070:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829884.661078:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829884/real 1787829884] req@ffff907d911483c0 x1874670011825152/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829884.661088:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011825152/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829884.661097:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829884.661101:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829884.661104:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011825152:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829884.661128:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011825280:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829884.661149:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829884.661157:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829884/real 1787829884] req@ffff907d9114b480 x1874670011825280/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829884.661166:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011825280/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829884.661171:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829884.661174:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829884.661176:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011825280:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829884.661181:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011825408:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829884.661191:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829884.661197:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829884/real 1787829884] req@ffff907d91149680 x1874670011825408/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829884.661204:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011825408/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829884.661210:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829884.661211:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829884.661213:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011825408:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829884.788702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829884.788756:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829885.812725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829885.812739:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829886.836757:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829886.836832:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829887.860746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829887.860775:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829888.884716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829888.884800:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829889.780783:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829889.780794:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829889.780798:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829889.780809:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5293 00000100:00080000:2.0:1787829889.780822:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829889.780830:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829889.780877:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011825536/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829889.780900:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829889.780903:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829889.780904:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829889.780907:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5293 00000100:00080000:2.0:1787829889.780910:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829889.780913:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829889.780923:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829889.780933:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011825664/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829889.780942:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011825536:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829889.780952:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829889.780956:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829889.780958:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829889.780961:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5293 00000100:00080000:2.0:1787829889.780963:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829889.780966:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829889.780969:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829889.780979:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011825792/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829889.780986:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829889.780989:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829889.780991:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829889.780993:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5293 00000100:00080000:2.0:1787829889.780995:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829889.780997:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829889.781000:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829889.781013:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011825920/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829889.781034:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829889.781092:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829889/real 1787829889] req@ffff907d91149680 x1874670011825536/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829889.781110:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011825536/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829889.781119:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829889.781122:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829889.781126:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011825536:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829889.781149:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011825664:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829889.781168:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829889.781181:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829889/real 1787829889] req@ffff907d9114b480 x1874670011825664/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829889.781194:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011825664/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829889.781216:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829889.781220:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829889.781224:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011825664:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829889.781231:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011825792:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829889.781252:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829889.781261:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829889/real 1787829889] req@ffff907d911483c0 x1874670011825792/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829889.781269:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011825792/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829889.781274:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829889.781277:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829889.781278:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011825792:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829889.781282:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011825920:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829889.781293:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829889.781298:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829889/real 1787829889] req@ffff907d9114bc00 x1874670011825920/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829889.781306:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011825920/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829889.781312:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829889.781313:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829889.781315:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011825920:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829889.908723:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829889.908796:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829890.932708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829890.932800:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829891.956699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829891.956794:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829892.980714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829892.980773:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829894.004717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829894.004785:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829894.900718:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829894.900733:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829894.900738:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829894.900753:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5298 00000100:00080000:1.0:1787829894.900768:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829894.900777:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829894.900839:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011826048/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829894.900871:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829894.900875:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829894.900877:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829894.900881:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5298 00000100:00080000:1.0:1787829894.900886:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829894.900891:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829894.900903:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829894.900914:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011826048:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829894.900919:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011826176/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829894.900934:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829894.900938:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829894.900941:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829894.900945:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5298 00000100:00080000:1.0:1787829894.900949:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829894.900953:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829894.900960:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829894.900972:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011826304/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829894.900984:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829894.900988:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829894.900990:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829894.900994:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5298 00000100:00080000:1.0:1787829894.900998:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829894.901002:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829894.901007:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787829894.901017:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829894.901021:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011826432/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829894.901065:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829894/real 1787829894] req@ffff907d908a3840 x1874670011826048/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829894.901085:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011826048/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829894.901097:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829894.901102:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829894.901107:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011826048:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829894.901143:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011826176:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829894.901166:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829894.901177:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829894/real 1787829894] req@ffff907d908a2580 x1874670011826176/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829894.901185:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011826176/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829894.901192:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829894.901194:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829894.901197:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011826176:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829894.901201:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011826304:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829894.901214:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829894.901220:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829894/real 1787829894] req@ffff907d908a2d00 x1874670011826304/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829894.901228:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011826304/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829894.901233:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829894.901235:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829894.901237:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011826304:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829894.901241:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011826432:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829894.901251:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829894.901257:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829894/real 1787829894] req@ffff907d908a2940 x1874670011826432/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829894.901269:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011826432/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829894.901275:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829894.901276:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829894.901278:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011826432:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829895.028700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829895.028740:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829896.052719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829896.052774:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829897.076748:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829897.076835:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829898.100787:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1787829898.100838:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:3.0:1787829899.124742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829899.124816:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829900.020739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829900.020752:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829900.020758:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829900.020769:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5303 00000100:00080000:1.0:1787829900.020783:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829900.020790:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829900.020832:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a0000 x1874670011826560/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829900.020860:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829900.020863:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829900.020865:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829900.020869:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5303 00000100:00080000:1.0:1787829900.020872:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829900.020876:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829900.020888:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829900.020902:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1e00 x1874670011826688/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829900.020912:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011826560:192.168.203.104@tcp:250:kworker.0 00000100:00080000:1.0:1787829900.020915:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829900.020922:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829900.020924:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829900.020929:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5303 00000100:00080000:1.0:1787829900.020931:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829900.020935:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829900.020940:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829900.020951:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2940 x1874670011826816/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829900.020965:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829900.020968:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1787829900.020970:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829900.020973:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5303 00000100:00080000:1.0:1787829900.020978:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829900.020981:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829900.020986:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1787829900.020992:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829900.021000:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2d00 x1874670011826944/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829900.021039:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829900/real 1787829900] req@ffff907d908a0000 x1874670011826560/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829900.021058:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a0000 x1874670011826560/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829900.021070:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829900.021073:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829900.021078:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011826560:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829900.021100:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011826688:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829900.021119:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829900.021130:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829900/real 1787829900] req@ffff907d908a1e00 x1874670011826688/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829900.021148:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1e00 x1874670011826688/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829900.021156:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829900.021159:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829900.021162:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011826688:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829900.021168:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011826816:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829900.021189:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829900.021199:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829900/real 1787829900] req@ffff907d908a2940 x1874670011826816/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829900.021210:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2940 x1874670011826816/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829900.021217:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829900.021220:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829900.021223:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011826816:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829900.021228:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011826944:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829900.021242:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829900.021251:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829900/real 1787829900] req@ffff907d908a2d00 x1874670011826944/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829900.021274:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2d00 x1874670011826944/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829900.021288:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829900.021291:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829900.021295:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011826944:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829900.148705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829900.148715:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829901.172709:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829901.172765:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829902.196734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787829902.196810:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829903.220785:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829903.220802:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:0.0:1787829904.244705:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829904.244793:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829905.140765:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829905.140778:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1787829905.140784:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829905.140797:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5308 00000100:00080000:1.0:1787829905.140811:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829905.140819:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829905.140868:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a2580 x1874670011827072/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829905.140897:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829905.140901:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1787829905.140903:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829905.140907:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5308 00000100:00080000:1.0:1787829905.140912:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829905.140917:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829905.140929:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829905.140939:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011827072:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829905.140944:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a3840 x1874670011827200/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829905.140957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829905.140961:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829905.140962:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829905.140967:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5308 00000100:00080000:1.0:1787829905.140971:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829905.140975:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829905.140980:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829905.140991:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a1680 x1874670011827328/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829905.141001:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829905.141006:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1787829905.141006:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:1.0:1787829905.141009:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829905.141015:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5308 00000100:00080000:1.0:1787829905.141019:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829905.141023:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:1.0:1787829905.141029:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829905.141041:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908a21c0 x1874670011827456/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829905.141067:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829905/real 1787829905] req@ffff907d908a2580 x1874670011827072/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829905.141088:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a2580 x1874670011827072/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829905.141099:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829905.141102:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829905.141105:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a2580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011827072:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829905.141126:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011827200:192.168.203.104@tcp:38:kworker.0 00000800:00000100:2.0:1787829905.141143:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829905.141151:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829905/real 1787829905] req@ffff907d908a3840 x1874670011827200/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829905.141159:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a3840 x1874670011827200/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829905.141168:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829905.141170:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829905.141172:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a3840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011827200:192.168.203.104@tcp:38:kworker.0 00000100:00100000:2.0:1787829905.141175:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011827328:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829905.141186:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829905.141191:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829905/real 1787829905] req@ffff907d908a1680 x1874670011827328/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829905.141198:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a1680 x1874670011827328/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829905.141206:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829905.141210:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829905.141213:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a1680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011827328:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829905.141218:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011827456:192.168.203.104@tcp:8:kworker.0 00000800:00000100:2.0:1787829905.141247:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829905.141257:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829905/real 1787829905] req@ffff907d908a21c0 x1874670011827456/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829905.141269:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908a21c0 x1874670011827456/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829905.141276:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829905.141279:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:2.0:1787829905.141282:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908a21c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011827456:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829905.268772:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829905.268793:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829906.292750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829906.292925:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829907.316744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829907.316793:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829908.340758:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829908.340821:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829909.364747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829909.364804:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829910.260750:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829910.260764:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829910.260769:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829910.260781:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5314 00000100:00080000:2.0:1787829910.260795:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829910.260804:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829910.260848:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011827584/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829910.260884:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829910.260888:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829910.260890:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829910.260894:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5314 00000100:00080000:2.0:1787829910.260898:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829910.260903:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829910.260915:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829910.260927:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011827712/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829910.260940:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829910.260944:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011827584:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829910.260945:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829910.260948:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829910.260959:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5314 00000100:00080000:2.0:1787829910.260963:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829910.260967:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829910.260972:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829910.260985:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011827840/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829910.260994:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829910.260997:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829910.261000:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829910.261011:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5314 00000100:00080000:2.0:1787829910.261018:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829910.261023:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829910.261028:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829910.261042:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011827968/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829910.261075:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011827712:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829910.261114:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011827840:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829910.261130:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011827968:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829910.388772:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829910.388793:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829911.412750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829911.412835:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829912.436788:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829912.436791:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787829913.332785:0:2336:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:0.0:1787829913.332815:0:2336:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1787829913.332827:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787829913.332856:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787829913.332868:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:0.0:1787829913.332880:0:2336:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:1.0:1787829913.332882:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829910/real 1787829913] req@ffff907d9114a580 x1874670011827712/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829913.332903:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011827712/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829913.332914:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829913.332920:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829913.332924:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011827712:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829913.332931:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829910/real 1787829913] req@ffff907d9114b840 x1874670011827840/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829913.332942:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011827840/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829913.332949:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829913.332953:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829913.332956:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011827840:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829913.332964:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829910/real 1787829913] req@ffff907d91149a40 x1874670011827968/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829913.332974:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011827968/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829913.332981:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829913.332983:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829913.332986:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011827968:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829913.332991:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829910/real 1787829913] req@ffff907d9114ad00 x1874670011827584/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829913.333001:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011827584/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829913.333007:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829913.333010:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829913.333013:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011827584:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829913.460735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829913.460789:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829914.484749:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829914.484777:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829915.380726:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829915.380736:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829915.380740:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829915.380751:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5319 00000100:00080000:3.0:1787829915.380762:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829915.380768:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829915.380797:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011828096/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829915.380825:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829915.380832:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829915.380835:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829915.380840:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5319 00000100:00080000:3.0:1787829915.380845:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829915.380851:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829915.380861:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829915.380869:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011828096:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829915.380882:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011828224/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829915.380900:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829915.380905:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829915.380910:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829915.380918:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5319 00000100:00080000:3.0:1787829915.380923:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829915.380930:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829915.380937:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1787829915.380945:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829915.380958:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011828352/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829915.380971:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829915.380975:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829915.380978:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829915.380984:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5319 00000100:00080000:3.0:1787829915.380990:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829915.380996:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829915.381003:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829915.381011:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829915/real 1787829915] req@ffff907da562d680 x1874670011828096/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829915.381021:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011828480/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829915.381035:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011828096/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829915.381049:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829915.381054:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829915.381058:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011828096:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829915.381089:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011828224:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829915.381125:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829915.381139:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829915/real 1787829915] req@ffff907da562f480 x1874670011828224/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829915.381163:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011828224/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829915.381173:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829915.381176:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829915.381180:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011828224:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829915.381186:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011828352:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829915.381207:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829915.381217:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829915/real 1787829915] req@ffff907da562ed00 x1874670011828352/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829915.381228:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011828352/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829915.381236:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829915.381240:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829915.381243:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011828352:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829915.381248:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011828480:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829915.381268:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829915.381278:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829915/real 1787829915] req@ffff907da562c780 x1874670011828480/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829915.381294:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011828480/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829915.381306:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829915.381308:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829915.381311:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011828480:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829915.508716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829915.508774:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829916.532753:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829916.532827:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829917.556692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829917.556787:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829918.580716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829918.580791:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829919.604739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829919.604890:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829920.500745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829920.500758:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829920.500763:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829920.500775:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5324 00000100:00080000:3.0:1787829920.500788:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829920.500796:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829920.500850:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011828608/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829920.500878:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829920.500882:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829920.500884:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829920.500889:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5324 00000100:00080000:3.0:1787829920.500897:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829920.500902:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829920.500914:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829920.500930:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011828736/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829920.500931:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011828608:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829920.500953:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829920.500957:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829920.500959:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829920.500963:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5324 00000100:00080000:3.0:1787829920.500967:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829920.500972:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829920.500977:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829920.500988:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011828864/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829920.500998:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829920.501001:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829920.501003:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787829920.501008:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:3.0:1787829920.501012:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5324 00000100:00080000:3.0:1787829920.501018:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829920.501023:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829920.501028:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829920.501039:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011828992/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829920.501051:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829920/real 1787829920] req@ffff907da562cf00 x1874670011828608/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829920.501066:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011828608/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829920.501075:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829920.501078:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829920.501081:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011828608:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829920.501102:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011828736:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829920.501132:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829920.501148:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829920/real 1787829920] req@ffff907da562e580 x1874670011828736/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829920.501159:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011828736/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829920.501168:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829920.501170:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829920.501173:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011828736:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829920.501177:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011828864:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829920.501191:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829920.501197:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829920/real 1787829920] req@ffff907da562d2c0 x1874670011828864/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829920.501204:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011828864/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829920.501209:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829920.501211:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829920.501213:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011828864:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829920.501216:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011828992:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829920.501224:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829920.501229:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829920/real 1787829920] req@ffff907da562fc00 x1874670011828992/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829920.501235:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011828992/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829920.501239:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829920.501241:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829920.501243:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011828992:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829920.628726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829920.628767:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829921.652731:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829921.652758:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829922.676736:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829922.676804:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829923.700670:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829923.700746:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829924.724684:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829924.724704:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829925.620766:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829925.620776:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829925.620780:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829925.620791:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5329 00000100:00080000:2.0:1787829925.620803:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829925.620810:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829925.620848:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011829120/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829925.620873:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829925.620875:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829925.620877:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829925.620880:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5329 00000100:00080000:2.0:1787829925.620882:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829925.620884:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829925.620893:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829925.620902:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011829248/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829925.620909:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829925.620912:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829925.620913:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829925.620916:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5329 00000100:00080000:2.0:1787829925.620917:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829925.620919:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829925.620922:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829925.620930:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011829376/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829925.620932:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011829120:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829925.620942:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829925.620945:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829925.620947:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829925.620949:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5329 00000100:00080000:2.0:1787829925.620950:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829925.620952:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829925.620954:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829925.620962:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011829504/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829925.621015:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829925.621071:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829925/real 1787829925] req@ffff907d911492c0 x1874670011829120/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829925.621104:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011829120/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829925.621114:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829925.621118:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829925.621122:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011829120:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829925.621135:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011829248:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829925.621158:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829925.621170:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829925/real 1787829925] req@ffff907d91148b40 x1874670011829248/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829925.621180:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011829248/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829925.621187:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829925.621190:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829925.621193:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011829248:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829925.621216:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011829376:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829925.621234:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829925.621244:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829925/real 1787829925] req@ffff907d9114a940 x1874670011829376/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829925.621254:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011829376/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829925.621262:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829925.621264:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829925.621267:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011829376:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829925.621272:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011829504:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829925.621295:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829925.621304:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829925/real 1787829925] req@ffff907d91148f00 x1874670011829504/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829925.621314:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011829504/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829925.621322:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829925.621324:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829925.621327:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011829504:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829925.748737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829925.748792:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829926.772700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829926.772777:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829927.796741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829927.796775:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829928.820745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829928.820787:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829929.844719:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829929.844830:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829930.740771:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829930.740782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829930.740787:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829930.740797:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5334 00000100:00080000:3.0:1787829930.740810:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829930.740816:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829930.740855:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011829632/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829930.740880:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829930.740883:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829930.740884:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829930.740887:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5334 00000100:00080000:3.0:1787829930.740889:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829930.740891:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829930.740900:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829930.740909:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011829760/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829930.740923:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787829930.740926:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011829632:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829930.740926:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829930.740928:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829930.740935:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5334 00000100:00080000:3.0:1787829930.740938:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829930.740940:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829930.740943:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829930.740951:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011829888/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829930.740957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829930.740960:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829930.740961:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829930.740963:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5334 00000100:00080000:3.0:1787829930.740965:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829930.740967:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829930.740970:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829930.740978:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011830016/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829930.741011:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829930.741072:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829930/real 1787829930] req@ffff907da562c3c0 x1874670011829632/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829930.741088:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011829632/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829930.741097:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829930.741101:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829930.741104:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011829632:192.168.203.104@tcp:250:kworker.0 00000100:00100000:0.0:1787829930.741126:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011829760:192.168.203.104@tcp:38:kworker.0 00000800:00000100:0.0:1787829930.741143:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829930.741150:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829930/real 1787829930] req@ffff907da562f0c0 x1874670011829760/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829930.741158:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011829760/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829930.741163:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829930.741165:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829930.741167:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011829760:192.168.203.104@tcp:38:kworker.0 00000100:00100000:0.0:1787829930.741170:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011829888:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829930.741180:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829930.741186:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829930/real 1787829930] req@ffff907da562c000 x1874670011829888/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829930.741192:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011829888/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829930.741197:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829930.741199:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829930.741200:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011829888:192.168.203.104@tcp:8:kworker.0 00000100:00100000:0.0:1787829930.741204:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011830016:192.168.203.104@tcp:8:kworker.0 00000800:00000100:0.0:1787829930.741213:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:0.0:1787829930.741217:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829930/real 1787829930] req@ffff907da562e940 x1874670011830016/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829930.741224:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011830016/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829930.741229:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829930.741230:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:0.0:1787829930.741232:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011830016:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829930.868740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829930.868811:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829931.892737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829931.892807:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829932.916760:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829932.916833:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829933.940858:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829933.941103:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787829934.964696:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829934.964993:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829935.860723:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829935.860732:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829935.860736:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829935.860746:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5339 00000100:00080000:0.0:1787829935.860755:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829935.860760:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829935.860793:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90847c00 x1874670011830144/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829935.860821:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829935.860825:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829935.860827:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829935.860830:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5339 00000100:00080000:0.0:1787829935.860834:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829935.860838:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829935.860847:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829935.860859:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846580 x1874670011830272/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829935.860870:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829935.860874:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829935.860878:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829935.860881:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011830144:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829935.860886:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5339 00000100:00080000:0.0:1787829935.860893:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829935.860898:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829935.860903:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829935.860915:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908470c0 x1874670011830400/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829935.860925:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829935.860928:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829935.860930:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829935.860934:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5339 00000100:00080000:0.0:1787829935.860939:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829935.860943:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829935.860947:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829935.860959:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844b40 x1874670011830528/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829935.860979:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829935.861052:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829935/real 1787829935] req@ffff907d90847c00 x1874670011830144/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829935.861073:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90847c00 x1874670011830144/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829935.861084:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829935.861089:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829935.861093:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90847c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011830144:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829935.861120:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011830272:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829935.861145:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829935.861158:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829935/real 1787829935] req@ffff907d90846580 x1874670011830272/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829935.861171:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846580 x1874670011830272/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829935.861179:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829935.861183:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829935.861186:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011830272:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829935.861192:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011830400:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829935.861214:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829935.861225:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829935/real 1787829935] req@ffff907d908470c0 x1874670011830400/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829935.861237:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908470c0 x1874670011830400/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829935.861245:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829935.861248:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829935.861251:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908470c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011830400:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829935.861258:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011830528:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829935.861276:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829935.861287:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829935/real 1787829935] req@ffff907d90844b40 x1874670011830528/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829935.861298:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844b40 x1874670011830528/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829935.861305:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829935.861308:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829935.861312:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011830528:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829935.988734:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829935.988781:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829937.012727:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829937.012761:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829938.036758:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829938.036885:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829939.060737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829939.060809:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829940.084745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829940.084828:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829940.980710:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829940.980720:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829940.980724:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829940.980734:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5344 00000100:00080000:0.0:1787829940.980746:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829940.980753:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829940.980782:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908443c0 x1874670011830656/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829940.980806:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829940.980809:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829940.980811:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829940.980815:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5344 00000100:00080000:0.0:1787829940.980818:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829940.980822:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829940.980831:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829940.980844:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844000 x1874670011830784/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829940.980851:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011830656:192.168.203.104@tcp:250:kworker.0 00000100:00080000:0.0:1787829940.980855:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829940.980859:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829940.980861:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829940.980864:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5344 00000100:00080000:0.0:1787829940.980866:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829940.980870:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829940.980874:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829940.980888:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845e00 x1874670011830912/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829940.980897:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829940.980900:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829940.980902:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829940.980904:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5344 00000100:00080000:0.0:1787829940.980907:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829940.980910:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:1.0:1787829940.980914:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:0.0:1787829940.980916:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829940.980927:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d908452c0 x1874670011831040/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829940.980960:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829940/real 1787829940] req@ffff907d908443c0 x1874670011830656/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829940.980988:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908443c0 x1874670011830656/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829940.980999:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829940.981003:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829940.981007:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908443c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011830656:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829940.981028:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011830784:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829940.981049:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829940.981059:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829940/real 1787829940] req@ffff907d90844000 x1874670011830784/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829940.981070:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844000 x1874670011830784/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829940.981078:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829940.981081:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829940.981084:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011830784:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829940.981090:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011830912:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829940.981104:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829940.981112:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829940/real 1787829940] req@ffff907d90845e00 x1874670011830912/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829940.981122:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845e00 x1874670011830912/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829940.981130:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829940.981133:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829940.981136:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011830912:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829940.981140:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011831040:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829940.981152:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829940.981159:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829940/real 1787829940] req@ffff907d908452c0 x1874670011831040/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829940.981169:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d908452c0 x1874670011831040/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829940.981176:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829940.981179:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829940.981182:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d908452c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011831040:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829941.108708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829941.108773:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829942.132720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829942.132756:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829943.156747:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829943.156814:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829944.180717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829944.180750:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829945.204738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829945.204788:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829946.100747:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829946.100757:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829946.100762:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829946.100772:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5349 00000100:00080000:2.0:1787829946.100783:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829946.100790:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829946.100831:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011831168/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829946.100855:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829946.100857:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829946.100859:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829946.100861:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5349 00000100:00080000:2.0:1787829946.100863:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829946.100866:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829946.100875:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829946.100884:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011831296/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829946.100891:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829946.100893:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829946.100894:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829946.100896:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5349 00000100:00080000:2.0:1787829946.100898:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829946.100902:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:1.0:1787829946.100903:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011831168:192.168.203.104@tcp:250:kworker.0 00000008:00100000:2.0:1787829946.100905:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829946.100915:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011831424/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829946.100925:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829946.100928:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829946.100930:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829946.100932:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5349 00000100:00080000:2.0:1787829946.100933:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829946.100936:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829946.100950:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829946.100962:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011831552/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829946.100986:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829946.101033:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829946/real 1787829946] req@ffff907d91148f00 x1874670011831168/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829946.101046:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011831168/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829946.101054:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829946.101057:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829946.101060:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011831168:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829946.101079:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011831296:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829946.101095:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829946.101102:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829946/real 1787829946] req@ffff907d9114a940 x1874670011831296/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829946.101109:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011831296/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829946.101114:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829946.101116:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829946.101118:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011831296:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829946.101121:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011831424:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829946.101134:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829946.101139:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829946/real 1787829946] req@ffff907d91148b40 x1874670011831424/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829946.101146:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011831424/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829946.101151:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829946.101153:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829946.101154:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011831424:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829946.101157:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011831552:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829946.101166:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829946.101171:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829946/real 1787829946] req@ffff907d911492c0 x1874670011831552/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829946.101177:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011831552/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829946.101182:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829946.101183:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829946.101185:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011831552:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829946.228726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829946.228782:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829947.252742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829947.252823:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829948.276725:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829948.276810:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829949.300704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829949.300780:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829950.324700:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829950.324812:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829951.220693:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829951.220708:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829951.220714:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829951.220727:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5354 00000100:00080000:2.0:1787829951.220741:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829951.220749:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829951.220792:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011831680/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829951.220819:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829951.220824:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829951.220827:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829951.220832:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5354 00000100:00080000:2.0:1787829951.220836:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829951.220840:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829951.220852:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829951.220862:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011831680:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829951.220866:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011831808/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829951.220877:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829951.220880:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829951.220882:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829951.220886:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5354 00000100:00080000:2.0:1787829951.220889:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829951.220893:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829951.220898:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829951.220926:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011831936/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829951.220936:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829951.220939:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829951.220941:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829951.220944:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5354 00000100:00080000:2.0:1787829951.220948:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829951.220951:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829951.220956:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1787829951.220957:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:2.0:1787829951.220969:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011832064/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829951.221015:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829951/real 1787829951] req@ffff907d9114ad00 x1874670011831680/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829951.221035:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011831680/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829951.221046:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829951.221050:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829951.221054:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011831680:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829951.221083:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011831808:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829951.221121:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829951.221135:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829951/real 1787829951] req@ffff907d91149a40 x1874670011831808/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829951.221149:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011831808/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829951.221158:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829951.221161:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829951.221165:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011831808:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829951.221173:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011831936:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829951.221198:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829951.221212:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829951/real 1787829951] req@ffff907d9114b840 x1874670011831936/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829951.221233:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011831936/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829951.221244:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829951.221247:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829951.221251:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011831936:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829951.221266:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011832064:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829951.221291:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829951.221306:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829951/real 1787829951] req@ffff907d9114a580 x1874670011832064/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829951.221315:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011832064/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829951.221321:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829951.221324:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829951.221327:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011832064:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829951.348746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829951.348772:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829952.372726:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829952.372797:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829953.396740:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829953.396795:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829954.420699:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829954.420739:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829955.444753:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829955.444800:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829956.340772:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829956.340782:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829956.340786:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829956.340796:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5360 00000100:00080000:2.0:1787829956.340807:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829956.340814:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829956.340850:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011832192/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829956.340887:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829956.340891:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829956.340893:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829956.340898:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5360 00000100:00080000:2.0:1787829956.340902:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829956.340906:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829956.340916:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829956.340932:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011832320/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829956.340946:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011832192:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829956.340950:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829956.340960:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829956.340963:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829956.340968:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5360 00000100:00080000:2.0:1787829956.340972:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829956.340977:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829956.340982:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829956.340997:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011832448/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829956.341007:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829956.341011:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829956.341013:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829956.341017:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5360 00000100:00080000:2.0:1787829956.341029:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829956.341034:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000800:00000100:1.0:1787829956.341042:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000008:00100000:2.0:1787829956.341044:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829956.341060:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011832576/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829956.341097:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829956/real 1787829956] req@ffff907d9114bc00 x1874670011832192/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829956.341119:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011832192/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829956.341130:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829956.341134:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829956.341139:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011832192:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829956.341165:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011832320:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829956.341188:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829956.341213:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829956/real 1787829956] req@ffff907d911483c0 x1874670011832320/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829956.341225:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011832320/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829956.341232:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829956.341236:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829956.341239:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011832320:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829956.341245:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011832448:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829956.341264:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829956.341274:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829956/real 1787829956] req@ffff907d9114b480 x1874670011832448/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829956.341284:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011832448/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829956.341296:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829956.341299:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829956.341302:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011832448:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829956.341307:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011832576:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829956.341322:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829956.341333:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829956/real 1787829956] req@ffff907d91149680 x1874670011832576/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829956.341343:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011832576/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829956.341350:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829956.341353:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829956.341355:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011832576:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829956.468695:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829956.468762:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829957.492718:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829957.492857:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829958.516704:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829958.516755:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829959.540708:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829959.540772:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829960.564780:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829960.564827:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829961.460755:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829961.460763:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829961.460767:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829961.460775:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5365 00000100:00080000:2.0:1787829961.460784:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829961.460789:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829961.460821:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149680 x1874670011832704/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829961.460864:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829961.460868:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829961.460870:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829961.460875:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5365 00000100:00080000:2.0:1787829961.460878:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829961.460883:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829961.460893:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829961.460909:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b480 x1874670011832832/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829961.460914:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011832704:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829961.460927:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829961.460930:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829961.460932:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829961.460937:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5365 00000100:00080000:2.0:1787829961.460942:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829961.460946:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829961.460951:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829961.460961:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911483c0 x1874670011832960/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829961.460967:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787829961.460971:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787829961.460972:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829961.460975:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829961.460978:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5365 00000100:00080000:2.0:1787829961.460981:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829961.460983:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829961.460986:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829961.460994:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114bc00 x1874670011833088/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829961.461014:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829961/real 1787829961] req@ffff907d91149680 x1874670011832704/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829961.461033:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149680 x1874670011832704/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829961.461043:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829961.461047:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829961.461051:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011832704:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829961.461072:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011832832:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829961.461093:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829961.461115:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829961/real 1787829961] req@ffff907d9114b480 x1874670011832832/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829961.461128:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b480 x1874670011832832/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829961.461135:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829961.461137:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829961.461139:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011832832:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829961.461144:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011832960:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829961.461158:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829961.461165:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829961/real 1787829961] req@ffff907d911483c0 x1874670011832960/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829961.461172:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911483c0 x1874670011832960/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829961.461177:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829961.461178:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829961.461180:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911483c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011832960:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829961.461188:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011833088:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829961.461198:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829961.461203:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829961/real 1787829961] req@ffff907d9114bc00 x1874670011833088/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829961.461210:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114bc00 x1874670011833088/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829961.461214:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829961.461216:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829961.461217:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114bc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011833088:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829961.588717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829961.588760:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829962.612748:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829962.612917:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829963.636812:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829963.636847:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787829964.660746:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829964.660833:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829965.684861:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829965.684936:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829966.580786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829966.580810:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829966.580815:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829966.580829:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5370 00000100:00080000:3.0:1787829966.580844:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829966.580852:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829966.580911:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011833216/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829966.580937:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829966.580940:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829966.580942:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829966.580946:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5370 00000100:00080000:3.0:1787829966.580949:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829966.580953:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829966.580964:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829966.580977:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011833344/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829966.580986:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829966.580990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829966.580991:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829966.580996:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5370 00000100:00080000:3.0:1787829966.580999:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829966.581003:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829966.581011:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787829966.581014:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011833216:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829966.581028:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011833472/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829966.581040:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829966.581043:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829966.581045:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829966.581050:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5370 00000100:00080000:3.0:1787829966.581054:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829966.581057:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829966.581072:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829966.581083:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011833600/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829966.581111:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829966.581154:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829966/real 1787829966] req@ffff907da562f840 x1874670011833216/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829966.581172:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011833216/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829966.581183:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829966.581188:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829966.581192:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011833216:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829966.581199:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011833344:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829966.581229:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829966.581241:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829966/real 1787829966] req@ffff907da562da40 x1874670011833344/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829966.581254:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011833344/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829966.581262:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829966.581265:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829966.581269:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011833344:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829966.581293:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011833472:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829966.581314:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829966.581326:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829966/real 1787829966] req@ffff907da562e1c0 x1874670011833472/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829966.581338:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011833472/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829966.581346:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829966.581349:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829966.581352:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011833472:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829966.581359:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011833600:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829966.581378:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829966.581388:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829966/real 1787829966] req@ffff907da562de00 x1874670011833600/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829966.581400:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011833600/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829966.581410:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829966.581413:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829966.581416:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011833600:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829966.708680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829966.708774:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829967.732781:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829967.732881:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787829968.756711:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829968.756762:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829969.781129:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829969.781417:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:0.0:1787829970.804759:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829970.804883:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829971.700722:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829971.700732:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829971.700736:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829971.700745:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5375 00000100:00080000:3.0:1787829971.700756:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829971.700772:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829971.700806:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cb40 x1874670011833728/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829971.700836:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829971.700840:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829971.700842:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829971.700845:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5375 00000100:00080000:3.0:1787829971.700849:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829971.700852:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829971.700862:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829971.700875:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562de00 x1874670011833856/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829971.700883:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829971.700886:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829971.700888:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829971.700891:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5375 00000100:00080000:3.0:1787829971.700895:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829971.700898:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829971.700902:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829971.700913:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e1c0 x1874670011833984/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829971.700921:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829971.700924:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829971.700926:0:37:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829971.700928:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5375 00000100:00080000:3.0:1787829971.700932:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829971.700935:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829971.700939:0:37:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829971.700948:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562da40 x1874670011834112/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829971.701942:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011833728:192.168.203.104@tcp:250:kworker.0 00000800:00000100:1.0:1787829971.702007:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829971.702056:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829971/real 1787829971] req@ffff907da562cb40 x1874670011833728/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829971.702074:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cb40 x1874670011833728/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829971.702086:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829971.702090:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829971.702094:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011833728:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829971.702101:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011833856:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829971.702122:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829971.702134:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829971/real 1787829971] req@ffff907da562de00 x1874670011833856/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829971.702152:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562de00 x1874670011833856/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829971.702160:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829971.702163:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829971.702166:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011833856:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829971.702171:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011833984:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829971.702200:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829971.702212:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829971/real 1787829971] req@ffff907da562e1c0 x1874670011833984/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829971.702222:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e1c0 x1874670011833984/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829971.702228:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829971.702231:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829971.702234:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011833984:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829971.702240:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011834112:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829971.702265:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829971.702274:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829971/real 1787829971] req@ffff907da562da40 x1874670011834112/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829971.702285:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562da40 x1874670011834112/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829971.702303:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829971.702307:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829971.702310:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011834112:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787829971.828935:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829971.828984:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829972.852816:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829972.853002:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829973.876680:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829973.876810:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829974.900712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829974.901017:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829975.924777:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829975.924831:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829976.821636:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829976.821647:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1787829976.821653:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829976.821664:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5380 00000100:00080000:0.0:1787829976.821677:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829976.821685:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:0.0:1787829976.821741:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90844780 x1874670011834240/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829976.821780:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829976.821784:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1787829976.821786:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829976.821789:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5380 00000100:00080000:0.0:1787829976.821794:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829976.821798:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829976.821810:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829976.821823:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845680 x1874670011834368/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829976.821833:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829976.821836:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1787829976.821838:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829976.821842:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5380 00000100:00080000:0.0:1787829976.821845:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829976.821848:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829976.821853:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829976.821863:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90846d00 x1874670011834496/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787829976.821872:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829976.821875:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787829976.821877:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829976.821880:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5380 00000100:00080000:0.0:1787829976.821883:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1787829976.821887:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:0.0:1787829976.821892:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787829976.821901:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d90845a40 x1874670011834624/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829976.822171:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011834240:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829976.822331:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011834368:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829976.822359:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011834496:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829976.822378:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011834624:192.168.203.104@tcp:8:kworker.0 00010000:00010000:1.0:1787829976.948804:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1787829976.948894:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787829977.972743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829977.972872:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787829978.996898:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829978.997574:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000400:00000100:1.0:1787829979.893231:0:2337:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.104:988 00000400:00000100:1.0:1787829979.893276:0:2337:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.104@tcp at host 192.168.203.104 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829979.893292:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829979.893335:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829979.893355:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000800:00000100:1.0:1787829979.893368:0:2337:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.104@tcp 00000100:00100000:3.0:1787829979.894645:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829976/real 1787829979] req@ffff907d90844780 x1874670011834240/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829979.894678:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90844780 x1874670011834240/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829979.894692:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829979.894697:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829979.894702:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90844780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011834240:192.168.203.104@tcp:250:kworker.0 00000100:00100000:3.0:1787829979.894724:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829976/real 1787829979] req@ffff907d90845680 x1874670011834368/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829979.894738:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845680 x1874670011834368/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829979.894746:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829979.894749:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829979.894753:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011834368:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829979.894759:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829976/real 1787829979] req@ffff907d90846d00 x1874670011834496/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829979.894772:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90846d00 x1874670011834496/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829979.894780:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829979.894784:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829979.894787:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90846d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011834496:192.168.203.104@tcp:8:kworker.0 00000100:00100000:3.0:1787829979.894795:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829976/real 1787829979] req@ffff907d90845a40 x1874670011834624/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829979.894805:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d90845a40 x1874670011834624/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829979.894812:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829979.894815:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829979.894819:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d90845a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011834624:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829980.020761:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829980.020863:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829981.044743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829981.045331:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829981.941549:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829981.941560:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829981.941564:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829981.941576:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5385 00000100:00080000:2.0:1787829981.941588:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829981.941596:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829981.941650:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a580 x1874670011834752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829981.941679:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829981.941683:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829981.941684:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829981.941688:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5385 00000100:00100000:3.0:1787829981.941835:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011834752:192.168.203.104@tcp:250:kworker.0 00000800:00000100:3.0:1787829981.941906:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00080000:2.0:1787829981.941929:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829981.941935:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829981.941945:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829981.941946:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829981/real 1787829981] req@ffff907d9114a580 x1874670011834752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829981.941963:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114b840 x1874670011834880/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829981.941975:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787829981.941978:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a580 x1874670011834752/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829981.941979:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829981.941982:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829981.941990:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5385 00000100:00080000:2.0:1787829981.941996:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829981.942000:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829981.942002:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00080000:3.0:1787829981.942009:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000008:00100000:2.0:1787829981.942010:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829981.942016:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011834752:192.168.203.104@tcp:250:kworker.0 00000100:00100000:2.0:1787829981.942027:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91149a40 x1874670011835008/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829981.942036:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787829981.942046:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011834880:192.168.203.104@tcp:38:kworker.0 00000800:00000100:3.0:1787829981.942071:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829981.942084:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829981/real 1787829981] req@ffff907d9114b840 x1874670011834880/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829981.942094:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114b840 x1874670011834880/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829981.942102:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829981.942104:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829981.942108:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114b840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011834880:192.168.203.104@tcp:38:kworker.0 00000100:00100000:3.0:1787829981.942113:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011835008:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829981.942131:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829981.942141:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829981/real 1787829981] req@ffff907d91149a40 x1874670011835008/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829981.942150:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91149a40 x1874670011835008/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829981.942157:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829981.942160:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829981.942162:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91149a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011835008:192.168.203.104@tcp:8:kworker.0 00000100:00080000:2.0:1787829981.942521:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829981.942523:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829981.942528:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5385 00000100:00080000:2.0:1787829981.942533:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829981.942537:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829981.942543:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829981.942559:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114ad00 x1874670011835136/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829981.943095:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011835136:192.168.203.104@tcp:8:kworker.0 00000800:00000100:3.0:1787829981.943127:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:3.0:1787829981.943143:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829981/real 1787829981] req@ffff907d9114ad00 x1874670011835136/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829981.943155:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114ad00 x1874670011835136/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829981.943164:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829981.943168:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:3.0:1787829981.943172:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114ad00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011835136:192.168.203.104@tcp:8:kworker.0 00010000:00010000:2.0:1787829982.068867:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829982.069329:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787829983.092755:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829983.092834:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787829984.116742:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:0.0:1787829984.116786:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1787829985.140920:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829985.142923:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787829986.164735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829986.164851:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829987.063482:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829987.063495:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829987.063500:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829987.063512:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5390 00000100:00080000:3.0:1787829987.063527:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829987.063535:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829987.063576:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f840 x1874670011835264/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829987.063633:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829987.063637:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829987.063651:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829987.063663:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5390 00000100:00080000:3.0:1787829987.063667:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829987.063671:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829987.063682:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829987.063696:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e940 x1874670011835392/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829987.063705:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829987.063708:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829987.063710:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829987.063714:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5390 00000100:00080000:3.0:1787829987.063718:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829987.063722:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829987.063727:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829987.063736:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c000 x1874670011835520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829987.063745:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829987.063748:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829987.063749:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829987.063753:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5390 00000100:00080000:3.0:1787829987.063756:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829987.063760:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829987.063764:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829987.063774:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f0c0 x1874670011835648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829987.063858:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011835264:192.168.203.104@tcp:250:kworker.0 00000800:00000100:1.0:1787829987.063933:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829987.064494:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829987/real 1787829987] req@ffff907da562f840 x1874670011835264/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829987.064522:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f840 x1874670011835264/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829987.064533:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829987.064537:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829987.064541:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011835264:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829987.064549:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011835392:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829987.064583:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829987.064596:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829987/real 1787829987] req@ffff907da562e940 x1874670011835392/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829987.064672:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e940 x1874670011835392/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829987.064680:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829987.064683:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829987.064687:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011835392:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829987.064693:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011835520:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829987.064717:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829987.064733:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829987/real 1787829987] req@ffff907da562c000 x1874670011835520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829987.064756:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c000 x1874670011835520/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829987.064770:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829987.064774:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829987.064777:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011835520:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829987.064784:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011835648:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829987.064808:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829987.064820:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829987/real 1787829987] req@ffff907da562f0c0 x1874670011835648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829987.064830:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f0c0 x1874670011835648/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829987.064838:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829987.064842:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829987.064845:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011835648:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829987.188741:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829987.188885:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829988.212724:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829988.212779:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829989.236733:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829989.236763:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829990.260782:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829990.262149:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1787829991.284969:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829991.285106:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00080000:2.0:1787829992.181067:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829992.181076:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787829992.181081:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829992.181090:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5395 00000100:00080000:2.0:1787829992.181102:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829992.181109:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787829992.181152:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011835776/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829992.181187:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829992.181192:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787829992.181193:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829992.181197:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5395 00000100:00080000:2.0:1787829992.181200:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829992.181205:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829992.181215:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829992.181228:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011835904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829992.181238:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829992.181241:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787829992.181244:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829992.181251:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5395 00000100:00100000:1.0:1787829992.181255:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011835776:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787829992.181256:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829992.181268:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829992.181274:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829992.181285:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011836032/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829992.181292:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829992.181295:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787829992.181296:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787829992.181299:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5395 00000100:00080000:2.0:1787829992.181303:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787829992.181306:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787829992.181311:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829992.181321:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011836160/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829992.181370:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829992.181436:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829992/real 1787829992] req@ffff907d911492c0 x1874670011835776/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829992.181457:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011835776/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829992.181469:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829992.181474:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829992.181478:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011835776:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829992.181506:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011835904:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829992.181532:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829992.181545:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829992/real 1787829992] req@ffff907d91148b40 x1874670011835904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829992.181558:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011835904/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829992.181566:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829992.181569:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829992.181572:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011835904:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829992.181578:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011836032:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829992.181600:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829992.181648:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829992/real 1787829992] req@ffff907d9114a940 x1874670011836032/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829992.181660:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011836032/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829992.181668:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829992.181671:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829992.181675:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011836032:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829992.181682:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011836160:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829992.181720:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829992.181733:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829992/real 1787829992] req@ffff907d91148f00 x1874670011836160/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829992.181745:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011836160/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829992.181758:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829992.181762:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829992.181772:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011836160:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829992.308745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829992.308757:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829993.332717:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829993.332819:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829994.356761:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829994.356925:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829995.380735:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829995.380778:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829996.404714:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829996.404766:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829997.300722:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829997.300734:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787829997.300740:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829997.300751:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5401 00000100:00080000:3.0:1787829997.300766:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829997.300773:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787829997.300815:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c3c0 x1874670011836288/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829997.300838:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829997.300841:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787829997.300842:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829997.300845:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5401 00000100:00080000:3.0:1787829997.300847:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829997.300849:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829997.300858:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829997.300871:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011836416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829997.300884:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829997.300888:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787829997.300892:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829997.300897:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011836288:192.168.203.104@tcp:250:kworker.0 00000100:00080000:3.0:1787829997.300903:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5401 00000100:00080000:3.0:1787829997.300907:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829997.300911:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829997.300915:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829997.300929:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011836544/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829997.300936:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829997.300939:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787829997.300941:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829997.300944:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5401 00000100:00080000:3.0:1787829997.300947:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829997.300949:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787829997.300953:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829997.300960:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011836672/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829997.300972:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829997.301015:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829997/real 1787829997] req@ffff907da562c3c0 x1874670011836288/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829997.301034:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c3c0 x1874670011836288/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829997.301044:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829997.301049:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829997.301052:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011836288:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787829997.301074:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011836416:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787829997.301092:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829997.301100:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829997/real 1787829997] req@ffff907da562fc00 x1874670011836416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829997.301108:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011836416/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829997.301113:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829997.301115:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829997.301117:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011836416:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787829997.301121:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011836544:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829997.301131:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829997.301136:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829997/real 1787829997] req@ffff907da562d2c0 x1874670011836544/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829997.301142:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011836544/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829997.301147:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829997.301149:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829997.301151:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011836544:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829997.301154:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011836672:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787829997.301163:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787829997.301168:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829997/real 1787829997] req@ffff907da562e580 x1874670011836672/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829997.301175:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011836672/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829997.301180:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829997.301181:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787829997.301183:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011836672:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787829997.428737:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787829997.428804:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829998.452753:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829998.452794:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787829999.476738:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787829999.476806:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830000.500843:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830000.503702:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:1.0:1787830001.524745:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787830001.524816:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787830002.420871:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830002.420886:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1787830002.420891:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830002.420903:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5406 00000100:00080000:2.0:1787830002.420918:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830002.420928:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:2.0:1787830002.420983:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148f00 x1874670011836800/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830002.421020:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830002.421024:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1787830002.421026:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830002.421031:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5406 00000100:00080000:2.0:1787830002.421035:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830002.421039:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787830002.421052:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787830002.421068:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d9114a940 x1874670011836928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830002.421081:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787830002.421082:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011836800:192.168.203.104@tcp:250:kworker.0 00000100:00080000:2.0:1787830002.421086:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1787830002.421090:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830002.421096:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5406 00000100:00080000:2.0:1787830002.421100:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830002.421104:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787830002.421109:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787830002.421121:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d91148b40 x1874670011837056/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787830002.421131:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787830002.421134:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1787830002.421137:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1787830002.421140:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5406 00000100:00080000:2.0:1787830002.421144:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1787830002.421148:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:2.0:1787830002.421153:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787830002.421165:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907d911492c0 x1874670011837184/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787830002.421176:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787830002.421254:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830002/real 1787830002] req@ffff907d91148f00 x1874670011836800/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830002.421274:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148f00 x1874670011836800/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830002.421286:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830002.421290:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787830002.421294:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011836800:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787830002.421322:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011836928:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787830002.421347:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787830002.421359:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830002/real 1787830002] req@ffff907d9114a940 x1874670011836928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830002.421375:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d9114a940 x1874670011836928/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830002.421382:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830002.421386:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787830002.421389:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d9114a940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011836928:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787830002.421395:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011837056:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787830002.421418:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787830002.421430:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830002/real 1787830002] req@ffff907d91148b40 x1874670011837056/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830002.421441:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d91148b40 x1874670011837056/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830002.421455:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830002.421458:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787830002.421462:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d91148b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011837056:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787830002.421467:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011837184:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787830002.421496:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787830002.421507:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830002/real 1787830002] req@ffff907d911492c0 x1874670011837184/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830002.421518:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907d911492c0 x1874670011837184/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830002.421530:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830002.421533:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787830002.421536:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907d911492c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011837184:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787830002.548753:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787830002.548802:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1787830003.572924:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830003.573309:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00000100:00100000:2.0:1787830004.596743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787830004.597778:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830005.620739:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787830005.620830:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830006.644764:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787830006.644887:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830007.540823:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830007.540837:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830007.540843:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830007.540855:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5411 00000100:00080000:3.0:1787830007.540872:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830007.540881:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787830007.540930:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011837312/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830007.540971:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830007.540975:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830007.540976:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830007.540980:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5411 00000100:00080000:3.0:1787830007.540984:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830007.540988:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787830007.541000:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830007.541013:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011837440/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830007.541022:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830007.541025:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830007.541027:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830007.541031:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5411 00000100:00080000:3.0:1787830007.541034:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830007.541038:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787830007.541042:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787830007.541052:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011837568/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830007.541060:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830007.541064:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830007.541065:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830007.541069:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5411 00000100:00080000:3.0:1787830007.541071:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830007.541075:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787830007.541078:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787830007.541087:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011837696/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830007.541197:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011837312:192.168.203.104@tcp:250:kworker.0 00000800:00000100:1.0:1787830007.541272:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787830007.541330:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830007/real 1787830007] req@ffff907da562cf00 x1874670011837312/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830007.541349:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011837312/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830007.541360:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830007.541364:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787830007.541368:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011837312:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787830007.541375:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011837440:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787830007.541398:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787830007.541412:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830007/real 1787830007] req@ffff907da562c780 x1874670011837440/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830007.541423:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011837440/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830007.541431:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830007.541434:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787830007.541437:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011837440:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787830007.541443:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011837568:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787830007.541463:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787830007.541473:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830007/real 1787830007] req@ffff907da562ed00 x1874670011837568/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830007.541484:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011837568/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830007.541491:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830007.541494:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787830007.541497:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011837568:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787830007.541502:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011837696:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787830007.541532:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787830007.541543:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830007/real 1787830007] req@ffff907da562f480 x1874670011837696/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830007.541553:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011837696/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830007.541561:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830007.541564:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787830007.541567:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011837696:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787830007.668750:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787830007.671215:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830008.692698:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787830008.692782:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830009.716712:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787830009.716749:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830010.740707:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787830010.740868:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830011.764692:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787830011.764836:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830012.660767:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830012.660779:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830012.660784:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830012.660797:0:70801:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5416 00000100:00080000:3.0:1787830012.660811:0:70801:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830012.660820:0:70801:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787830012.660875:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d680 x1874670011837824/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830012.660912:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830012.660920:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830012.660921:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830012.660927:0:70801:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5416 00000100:00080000:3.0:1787830012.660931:0:70801:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830012.660934:0:70801:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787830012.660947:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830012.660963:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562f480 x1874670011837952/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830012.660974:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830012.660983:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830012.660985:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830012.660989:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5416 00000100:00080000:3.0:1787830012.660995:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830012.660999:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787830012.661005:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787830012.661018:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562ed00 x1874670011838080/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830012.661027:0:70801:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830012.661030:0:70801:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830012.661031:0:70801:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830012.661034:0:70801:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5416 00000100:00100000:1.0:1787830012.661098:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011837824:192.168.203.104@tcp:250:kworker.0 00000800:00000100:1.0:1787830012.661181:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787830012.661231:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830012/real 1787830012] req@ffff907da562d680 x1874670011837824/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830012.661249:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d680 x1874670011837824/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830012.661260:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830012.661263:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787830012.661267:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011837824:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787830012.661285:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011837952:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787830012.661310:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787830012.661323:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830012/real 1787830012] req@ffff907da562f480 x1874670011837952/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830012.661335:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562f480 x1874670011837952/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830012.661343:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830012.661345:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787830012.661349:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011837952:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787830012.661354:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011838080:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787830012.661374:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787830012.661384:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830012/real 1787830012] req@ffff907da562ed00 x1874670011838080/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830012.661396:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562ed00 x1874670011838080/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830012.661404:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830012.661407:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787830012.661410:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011838080:192.168.203.104@tcp:8:kworker.0 00000100:00080000:3.0:1787830012.661491:0:70801:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830012.661497:0:70801:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787830012.661503:0:70801:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787830012.661516:0:70801:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562c780 x1874670011838208/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830012.661590:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011838208:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787830012.661643:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787830012.661654:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830012/real 1787830012] req@ffff907da562c780 x1874670011838208/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830012.661666:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562c780 x1874670011838208/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830012.661674:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830012.661677:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787830012.661681:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011838208:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787830012.788716:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787830012.791825:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830013.812744:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787830013.813106:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830014.836756:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787830014.836847:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830015.861620:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787830015.861820:0:60472:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830016.884743:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787830016.885098:0:40347:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787830017.784006:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) f411759c-2893-4f83-9e11-e7e6f379c36e->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830017.784016:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1787830017.784020:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830017.784029:0:70545:0:(import.c:510:import_select_connection()) MGC192.168.203.104@tcp: connect to NID 192.168.203.104@tcp last attempt 5421 00000100:00080000:3.0:1787830017.784040:0:70545:0:(import.c:589:import_select_connection()) MGC192.168.203.104@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830017.784047:0:70545:0:(import.c:624:import_select_connection()) MGC192.168.203.104@tcp: import ffff907d8351c000 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000100:00100000:3.0:1787830017.784081:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562cf00 x1874670011838336/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830017.784109:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830017.784114:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1787830017.784116:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830017.784120:0:70545:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5421 00000100:00080000:3.0:1787830017.784123:0:70545:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830017.784127:0:70545:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff907d986d0000: import ffff907d8351e800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787830017.784137:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787830017.784149:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562e580 x1874670011838464/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830017.784160:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830017.784163:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1787830017.784165:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830017.784168:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5421 00000100:00080000:3.0:1787830017.784172:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830017.784175:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff907d986d0000: import ffff907d98adc800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787830017.784180:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787830017.784189:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562d2c0 x1874670011838592/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787830017.784198:0:70545:0:(pinger.c:229:ptlrpc_pinger_process_import()) 6f891bcb-15cd-41e0-96e2-db843619c1be->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787830017.784201:0:70545:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1787830017.784203:0:70545:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787830017.784207:0:70545:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: connect to NID 192.168.203.104@tcp last attempt 5421 00000100:00080000:3.0:1787830017.784209:0:70545:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787830017.784213:0:70545:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff907d986d0000: import ffff907d866bb800 using connection 192.168.203.104@tcp/192.168.203.104@tcp 00000008:00100000:3.0:1787830017.784218:0:70545:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787830017.784226:0:70545:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff907da562fc00 x1874670011838720/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787830017.784468:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011838336:192.168.203.104@tcp:250:kworker.0 00000800:00000100:1.0:1787830017.784531:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787830017.784570:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830017/real 1787830017] req@ffff907da562cf00 x1874670011838336/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830017.784588:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562cf00 x1874670011838336/t0(0) o250->MGC192.168.203.104@tcp@192.168.203.104@tcp:26/25 lens 520/544 e 0 to 1 dl 1787830042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830017.784599:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351c000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830017.784603:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787830017.784635:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562cf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f411759c-2893-4f83-9e11-e7e6f379c36e:2347:1874670011838336:192.168.203.104@tcp:250:kworker.0 00000100:00100000:1.0:1787830017.784643:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011838464:192.168.203.104@tcp:38:kworker.0 00000800:00000100:1.0:1787830017.784667:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787830017.784680:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830017/real 1787830017] req@ffff907da562e580 x1874670011838464/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830017.784692:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562e580 x1874670011838464/t0(0) o38->lustre-MDT0000-mdc-ffff907d986d0000@192.168.203.104@tcp:12/10 lens 520/544 e 0 to 1 dl 1787830042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830017.784700:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d8351e800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830017.784703:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787830017.784706:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011838464:192.168.203.104@tcp:38:kworker.0 00000100:00100000:1.0:1787830017.784711:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011838592:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787830017.784730:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787830017.784741:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830017/real 1787830017] req@ffff907da562d2c0 x1874670011838592/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830017.784752:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562d2c0 x1874670011838592/t0(0) o8->lustre-OST0000-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830017.784760:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d98adc800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830017.784763:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787830017.784766:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011838592:192.168.203.104@tcp:8:kworker.0 00000100:00100000:1.0:1787830017.784773:0:2347:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011838720:192.168.203.104@tcp:8:kworker.0 00000800:00000100:1.0:1787830017.784792:0:2347:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.104@tcp 00000100:00100000:1.0:1787830017.784803:0:2347:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787830017/real 1787830017] req@ffff907da562fc00 x1874670011838720/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787830017.784814:0:2347:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff907da562fc00 x1874670011838720/t0(0) o8->lustre-OST0001-osc-ffff907d986d0000@192.168.203.104@tcp:28/4 lens 520/544 e 0 to 1 dl 1787830042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787830017.784822:0:2347:0:(import.c:65:import_set_state_nolock()) ffff907d866bb800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787830017.784825:0:2347:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.104@tcp failed (-110) 00000100:00100000:1.0:1787830017.784828:0:2347:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff907da562fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:6f891bcb-15cd-41e0-96e2-db843619c1be:2347:1874670011838720:192.168.203.104@tcp:8:kworker.0 00000100:00100000:2.0:1787830017.908720:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:1.0:1787830017.908943:0:47852:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830018.933059:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787830018.933667:0:40346:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830019.956788:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:3.0:1787830019.957185:0:60482:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1787830020.980850:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds 00010000:00010000:2.0:1787830020.981013:0:60835:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1787830022.004989:0:60483:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1787830022.005659:0:61297:0:(client.c:2715:ptlrpc_set_wait()) set ffff907da01f6900 going to sleep for 0 seconds Debug log: 78900 lines, 78900 kept, 0 dropped, 0 bad.